Add local pytest integration tests for the targetDB CI workflow

docs/getting_started.md

@@ -145,6 +145,46 @@ print(df.head())
 db.close()
 ```
 
+## Running Tests Locally
+
+The unit tests (`tests/test_*.py`, excluding `tests/integration/`) do not require a database
+and can be run at any time:
+
+```bash
+uv run pytest tests -m "not integration"
+```
+
+The integration tests (`tests/integration/`) exercise the same sequence of `pfs-targetdb-cli`
+commands as the `Test Database` GitHub Actions workflow
+(`.github/workflows/test_database.yml`): creating the database, installing the Q3C extension,
+creating the schema, and inserting the example data in `examples/data/`, this time also
+verifying the inserted rows with SQL queries. They require Docker (for a disposable
+PostgreSQL + Q3C container) and are skipped automatically, with a reason, if Docker is not
+available.
+
+```bash
+# Docker must be running (e.g. Docker Desktop or OrbStack on macOS)
+uv run pytest tests/integration -v
+```
+
+A dedicated PostgreSQL container is built from `tests/docker/Dockerfile` and started/stopped
+automatically for the test session by `docker compose` (`tests/docker/docker-compose.test.yml`).
+It listens on `localhost:15433` by default; set `TARGETDB_TEST_PG_PORT` to use a different port.
+Its data directory is a tmpfs mount, so nothing is left on disk once the tests finish, and all
+generated files (converted flux standard catalogs, transferred target lists) are written under a
+pytest-managed temporary directory rather than into `examples/data/`.
+
+If a test fails and you want to inspect the database afterwards, pass `--keep-db` to leave the
+container running:
+
+```bash
+uv run pytest tests/integration -v --keep-db
+psql -h localhost -p 15433 -U postgres -d test_targetdb
+
+# once done, stop the container manually
+docker compose -f tests/docker/docker-compose.test.yml down -v
+```
+
 ## Build the Documentation
 
 The documentation can be built by the following command:

docs/tbls/public.fluxstd.md

@@ -102,9 +102,9 @@
 | fluxstd_pkey | CREATE UNIQUE INDEX fluxstd_pkey ON public.fluxstd USING btree (fluxstd_id) |
 | uq_obj_id_input_catalog_id_version | CREATE UNIQUE INDEX uq_obj_id_input_catalog_id_version ON public.fluxstd USING btree (obj_id, input_catalog_id, version) |
 | fluxstd_q3c_ang2ipix_idx | CREATE INDEX fluxstd_q3c_ang2ipix_idx ON public.fluxstd USING btree (q3c_ang2ipix(ra, "dec")) |
+| ix_fluxstd_version | CREATE INDEX ix_fluxstd_version ON public.fluxstd USING btree (version) |
 | ix_fluxstd_input_catalog_fluxstdid | CREATE INDEX ix_fluxstd_input_catalog_fluxstdid ON public.fluxstd USING btree (input_catalog_id, fluxstd_id) |
 | ix_fluxstd_version_fluxstdid | CREATE INDEX ix_fluxstd_version_fluxstdid ON public.fluxstd USING btree (version, fluxstd_id) |
-| ix_fluxstd_version | CREATE INDEX ix_fluxstd_version ON public.fluxstd USING btree (version) |
 
 ## Relations
 

docs/tbls/public.sky.md

@@ -34,9 +34,9 @@
 | ---- | ---------- |
 | sky_pkey | CREATE UNIQUE INDEX sky_pkey ON public.sky USING btree (sky_id) |
 | sky_obj_id_input_catalog_id_version_key | CREATE UNIQUE INDEX sky_obj_id_input_catalog_id_version_key ON public.sky USING btree (obj_id, input_catalog_id, version) |
-| ix_sky_input_catalog_id | CREATE INDEX ix_sky_input_catalog_id ON public.sky USING btree (input_catalog_id) |
 | sky_q3c_ang2ipix_idx | CREATE INDEX sky_q3c_ang2ipix_idx ON public.sky USING btree (q3c_ang2ipix(ra, "dec")) |
 | ix_sky_version | CREATE INDEX ix_sky_version ON public.sky USING btree (version) |
+| ix_sky_input_catalog_id | CREATE INDEX ix_sky_input_catalog_id ON public.sky USING btree (input_catalog_id) |
 
 ## Relations
 

docs/tbls/public.target.md

@@ -121,11 +121,11 @@
 | target_pkey | CREATE UNIQUE INDEX target_pkey ON public.target USING btree (target_id) |
 | target_propid_obcode_catid_objid_resolution_key | CREATE UNIQUE INDEX target_propid_obcode_catid_objid_resolution_key ON public.target USING btree (proposal_id, ob_code, input_catalog_id, obj_id, is_medium_resolution) |
 | target_propid_obcode_key | CREATE UNIQUE INDEX target_propid_obcode_key ON public.target USING btree (proposal_id, ob_code) |
-| target_proposal_id_idx | CREATE INDEX target_proposal_id_idx ON public.target USING btree (proposal_id) |
-| target_obj_id_input_catalog_id_idx | CREATE INDEX target_obj_id_input_catalog_id_idx ON public.target USING btree (obj_id, input_catalog_id) |
 | target_input_catalog_id_idx | CREATE INDEX target_input_catalog_id_idx ON public.target USING btree (input_catalog_id) |
 | target_proposal_id_obj_id_idx | CREATE INDEX target_proposal_id_obj_id_idx ON public.target USING btree (proposal_id, obj_id) |
 | target_q3c_ang2ipix_idx | CREATE INDEX target_q3c_ang2ipix_idx ON public.target USING btree (q3c_ang2ipix(ra, "dec")) |
+| target_proposal_id_idx | CREATE INDEX target_proposal_id_idx ON public.target USING btree (proposal_id) |
+| target_obj_id_input_catalog_id_idx | CREATE INDEX target_obj_id_input_catalog_id_idx ON public.target USING btree (obj_id, input_catalog_id) |
 
 ## Relations
 

pyproject.toml

@@ -45,13 +45,19 @@ source = "https://github.com/Subaru-PFS/ets_target_database"
 pfs-targetdb-cli = "targetdb.cli.cli_main:app"
 
 [project.optional-dependencies]
-dev = ["ipython", "pytest", "black", "ruff", "setuptools-scm>=8.0"]
+dev = ["ipython", "pytest", "pytest-cov", "black", "ruff", "setuptools-scm>=8.0"]
 doc = ["mkdocs<2", "mkdocs-material[imaging]"]
 
 [build-system]
 requires = ["setuptools>=64", "setuptools-scm>=8.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+  "integration: tests requiring a PostgreSQL container via Docker",
+]
+
 [tool.black]
 line-length = 88
 

tests/conftest.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+import shutil
+import subprocess
+from functools import lru_cache
+from pathlib import Path
+
+import pytest
+
+INTEGRATION_TESTS_DIR = Path(__file__).parent / "integration"
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--keep-db",
+        action="store_true",
+        default=False,
+        help=(
+            "Do not tear down the PostgreSQL test container after the "
+            "integration test session. Useful for inspecting the database "
+            "after a failure, e.g. with "
+            "`psql -h localhost -p 15433 -U postgres -d test_targetdb`."
+        ),
+    )
+
+
+@lru_cache(maxsize=1)
+def docker_available_reason():
+    """
+    Check whether a usable Docker daemon is available.
+
+    Returns
+    -------
+    reason : str or None
+        None if Docker is available and responsive, otherwise a short
+        string explaining why it is not, suitable for a pytest skip reason.
+    """
+    if shutil.which("docker") is None:
+        return "docker executable not found on PATH"
+
+    try:
+        result = subprocess.run(
+            ["docker", "info"],
+            capture_output=True,
+            timeout=10,
+        )
+    except (OSError, subprocess.TimeoutExpired) as e:
+        return f"docker info failed to run: {e}"
+
+    if result.returncode != 0:
+        return "docker daemon is not reachable (`docker info` failed)"
+
+    return None
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers",
+        "integration: tests requiring a PostgreSQL container via Docker",
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    reason = docker_available_reason()
+
+    for item in items:
+        try:
+            item.path.relative_to(INTEGRATION_TESTS_DIR)
+        except ValueError:
+            continue
+
+        item.add_marker(pytest.mark.integration)
+
+        if reason is not None:
+            item.add_marker(pytest.mark.skip(reason=f"Docker unavailable: {reason}"))

tests/docker/Dockerfile

@@ -0,0 +1,9 @@
+# PostgreSQL image with the Q3C extension pre-installed, mirroring the
+# packages installed on-the-fly in .github/workflows/test_database.yml.
+# Building this once and reusing the image is much faster than the CI
+# approach of installing q3c into a freshly-started container every run.
+FROM postgres:16
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends postgresql-16-q3c \
+    && rm -rf /var/lib/apt/lists/*

tests/docker/docker-compose.test.yml

@@ -0,0 +1,26 @@
+# Standalone PostgreSQL + Q3C stack used by the pytest integration tests
+# (tests/integration/). Kept separate from examples/docker/, which is a
+# developer-facing, persistent local database and not meant to be
+# started/stopped/discarded by the test suite.
+name: targetdb-test
+
+services:
+  postgres:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
+    ports:
+      - "${TARGETDB_TEST_PG_PORT:-15433}:5432"
+    # Data lives in tmpfs: each test session starts from a clean database
+    # and nothing is left on disk after teardown.
+    tmpfs:
+      - /var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "postgres"]
+      interval: 2s
+      timeout: 5s
+      retries: 30