build PFF ACS 2024

dcpy/lifecycle/builds/utils.py

@@ -221,39 +221,34 @@ def read_recipe_df(
 
     last_error = None
     for file_type in file_types_to_try:
-        # Create temp file with appropriate extension
-        suffix = f".{file_type.to_extension()}"
-        with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
-            tmp_path = Path(tmp.name)
-
         try:
-            # Pull the file
-            result = connector.pull_versioned(
-                key=dataset.id,
-                version=dataset.version,
-                destination_path=tmp_path,
-                file_type=file_type,
-            )
-
-            # Read based on file type
-            if file_type == DatasetType.parquet:
-                return pd.read_parquet(result["path"])
-            elif file_type == DatasetType.csv:
-                return pd.read_csv(result["path"])
-            else:
-                raise ValueError(
-                    f"Unsupported file type for DataFrame reading: {file_type}"
+            # pull_versioned writes <filename> *inside* destination_path, so it
+            # must be a directory (as every other caller passes). TemporaryDirectory
+            # handles cleanup; read the file inside the `with`, before it's removed.
+            with tempfile.TemporaryDirectory() as tmp_dir:
+                result = connector.pull_versioned(
+                    key=dataset.id,
+                    version=dataset.version,
+                    destination_path=Path(tmp_dir),
+                    file_type=file_type,
                 )
 
+                # Read based on file type
+                if file_type == DatasetType.parquet:
+                    return pd.read_parquet(result["path"])
+                elif file_type == DatasetType.csv:
+                    return pd.read_csv(result["path"])
+                else:
+                    raise ValueError(
+                        f"Unsupported file type for DataFrame reading: {file_type}"
+                    )
+
         except Exception as e:
             last_error = e
             logger.debug(
                 f"Failed to read {dataset.id} v{dataset.version} as {file_type}: {e}"
             )
             continue
-        finally:
-            if tmp_path.exists():
-                tmp_path.unlink()
 
     # If we get here, all attempts failed
     if last_error:

dcpy/lifecycle/ingest/models.py

@@ -271,7 +271,13 @@ class ProcessingResult(SortedSerializedBase, arbitrary_types_allowed=True):
 
 
 class SparseConfig(BaseModel, extra="allow"):
-    id: str = Field(validation_alias=AliasChoices("id", AliasPath("dataset", "name")))
+    id: str = Field(
+        validation_alias=AliasChoices(
+            "id",  # ingest
+            "name",  # ingest - outdated
+            AliasPath("dataset", "name"),  # library
+        )
+    )
     version: str = Field(
         validation_alias=AliasChoices(
             "timestamp",  # ingest - raw

dcpy/test/lifecycle/builds/test_load.py

@@ -138,7 +138,6 @@ def test_df_from_df(self):
     @patch("dcpy.utils.postgres.PostgresClient")
     def test_df_from_pg(self, pg_client):
         _df = load.get_imported_df(self.load_result, "pg_dump")
-        print(pg_client.return_value.read_table_df)
         pg_client.return_value.read_table_df.assert_called_with("pg_dump")
 
     def test_df_from_csv(self):

products/factfinder/README.md

@@ -47,6 +47,66 @@ Code in this repo primarily:
 2. Runs pipelines to build Decennial and ACS datasets. 
    You'll probably want to run both `products/factfinder/run.py` for both 'acs' and 'decennial'
 
+### Command
+
+Run from the **repo root**, with the environment loaded (direnv, or an activated `.venv`, so
+`API_KEY` and the recipe credentials are set):
+
+```bash
+python3 -m products.factfinder.run <dataset> <version>
+# e.g. the 2019-2023 ACS update (recipe version 2024):
+python3 -m products.factfinder.run acs 2024
+```
+
+`<dataset>` is `acs` or `decennial`; `<version>` is the recipe version (e.g. `2024`).
+
+### Output
+
+Build artifacts land under `.lifecycle/` (gitignored):
+
+- `.lifecycle/builds/load/<recipe_id>/<version>/` — source data pulled from `edm-recipes`
+  (e.g. `dcp_pop_acs/2024/dcp_pop_acs.xlsx`)
+- `.lifecycle/builds/build/factfinder/<dataset>/<version>/` — the build output: `<dataset>.csv`
+  plus `metadata.json`
+
+So the 2024 ACS run writes `.lifecycle/builds/build/factfinder/acs/2024/acs.csv` — the local
+build output to QA before it's promoted/published.
+
+### Promote to Draft (manual)
+
+Unlike our other products, PFF is **not** wired into the `promote_to_draft.yml` automation — there
+is no command for this step. **We copy the build output up to S3 by hand.** Drafts live in the
+`edm-publishing` bucket under `db-factfinder/`, structured to mirror how every other product
+stores drafts:
+
+```
+edm-publishing/db-factfinder/draft/<dataset>/<version>/<release>_<n>/
+```
+
+- `<dataset>` — `acs` or `decennial`
+- `<version>` — the data version: ACS current-window end year (e.g. `2024`), previous window
+  (`2010`), or decennial (`2020`)
+- `<release>_<n>` — PFF release year and draft revision, e.g. `2026_2` is the second draft of the
+  2026 release
+
+Each draft folder holds `<dataset>.csv` and `metadata.json`, copied up from the local build output
+at `.lifecycle/builds/build/factfinder/<dataset>/<version>/`. ACS drafts also include a
+`metadata_diffs.txt` listing the variables added and dropped between this ACS year and the previous
+one — generate it with `qa/compare_acs_metadata.py <current_year> <previous_year>` (e.g.
+`2024 2023`).
+
+Examples from the 2026 release:
+
+- `edm-publishing/db-factfinder/draft/acs/2024/2026_2/`
+- `edm-publishing/db-factfinder/draft/acs/2010/2026_2/`
+- `edm-publishing/db-factfinder/draft/decennial/2020/2026_1/decennial.csv`
+
+**Note:** Application Engineering's (AE) scripts historically read from an older, separate
+location — `edm-publishing/db-factfinder/ar_build/acs/<version>/<date>/` (e.g.
+`.../acs/2024/2026-04-21/`). The `draft/` layout above was introduced to bring PFF in line with our
+other products; AE may still consume `ar_build/` until they switch over. `ar_build/` is
+transitional and should be removed once AE migrates to `draft/`.
+
 
 ## Cheatsheet On The Data Sources 
 

products/factfinder/acs.yml

@@ -1,5 +1,5 @@
 name: PFF - ACS
-version: 2023
+version: 2024
 product: db-factfinder
 inputs:
   dataset_defaults:
@@ -15,10 +15,9 @@ inputs:
     - name: dcp_pop_acs2010_social
       version: 20240524
     - name: dcp_pop_acs
+      version: 2024
       destination: file
       file_type: xlsx
-  missing_versions_strategy: find_latest
-
 
 stage_config:
   builds.build:
@@ -27,6 +26,3 @@ stage_config:
     connector_args:
       - name: acl
         value: public-read
-      - name: build_note
-        value_from:
-          env: "BUILD_NOTE"