diff --git a/.github/workflows/validate-data.yml b/.github/workflows/validate-data.yml index eb276b3b8630..67d7f8eb0bcf 100644 --- a/.github/workflows/validate-data.yml +++ b/.github/workflows/validate-data.yml @@ -16,16 +16,48 @@ on: jobs: self-validate: runs-on: ubuntu-latest + outputs: + validation_scope: ${{ steps.scope.outputs.scope }} steps: - uses: actions/checkout@v4 + with: + # The committed Pages dump contains over a million generated files. + # Start with the validator and its tests. Data is added only when + # the pull request actually changes a record. + sparse-checkout: | + .github + app + tests + sparse-checkout-cone-mode: true - uses: actions/setup-python@v5 with: python-version: "3.12" + - name: Select validation scope + id: scope + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + if [[ "${{ github.event_name }}" == "push" ]]; then + echo "scope=full" >> "$GITHUB_OUTPUT" + elif gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${{ github.event.pull_request.number }}/files" --jq '.[].filename' | grep -q '^data/'; then + echo "scope=full" >> "$GITHUB_OUTPUT" + else + echo "scope=tests" >> "$GITHUB_OUTPUT" + fi - name: Self-check (bundled validator) - run: python -m app.validate + shell: bash + run: | + if [[ "${{ steps.scope.outputs.scope }}" == "full" ]]; then + git sparse-checkout add data + python -m app.validate + else + python -m unittest discover -s tests -v + fi engine-validate: needs: self-validate + if: needs.self-validate.outputs.validation_scope == 'full' uses: GetTechAPI/TechEngine/.github/workflows/validate-data.yml@main with: data-ref: ${{ github.event.pull_request.head.sha || github.sha }} diff --git a/README.md b/README.md index c892cd9e5526..bc7a342c3e34 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,47 @@ # TechAPI -> **Curated, open dataset for consumer electronics specs.** Free, share-alike, machine-readable. +> Curated, open dataset for consumer electronics specifications and game metadata. [![validate-data](https://github.com/GetTechAPI/TechAPI/actions/workflows/validate-data.yml/badge.svg)](https://github.com/GetTechAPI/TechAPI/actions/workflows/validate-data.yml) - Data: **CC-BY-SA 4.0** +Data: **CC-BY-SA 4.0** -This repo holds the curated **dataset** and the **public site** (Astro intro + -playground). The API server, ingestion crawlers, coverage checks, and the -static-dump generator live in -[**TechEngine**](https://github.com/GetTechAPI/TechEngine). +This repository owns the versioned dataset and the public Astro site. The API +server, ingestion crawlers, coverage checks, and static-dump generator live in +[TechEngine](https://github.com/GetTechAPI/TechEngine), included here as the +`TechEngine` submodule. ## Layout ```text -data/brand//.json # e.g. data/brand/kr/samsung.json -data/soc///.json # data/soc/qualcomm/2024/snapdragon-8-elite.json -data/smartphone///.json # data/smartphone/samsung/2025/galaxy-s25.json -data/gpu////.json # data/gpu/nvidia/2025/consumer/geforce-rtx-5090.json -data/cpu////.json # data/cpu/intel/2023/consumer/core-i9-14900k.json +data/brand//.json +data/cpu////.json +data/gpu////.json +data/smartphone///.json +data/game//.json +site/ # Astro Pages site +site/public/v1/ # committed static API dump from TechEngine +app/validate.py # stdlib-only, fast repository self-check ``` -All paths use singular folder names. Slugs are kebab-case and unique within each category. +The game directory is sharded by normalized slug rather than genre or import +batch. See [the data layout guide](docs/DATA_LAYOUT.md) before adding or moving +game records. -The Astro site lives under `site/` and is the deploy target for GitHub Pages. It -consumes the static JSON dump produced by TechEngine's `weekly-refresh` workflow. - -## Self-Check - -A lightweight bundled validator lives at `app/validate.py`. It runs on every PR via -[`validate-data.yml`](.github/workflows/validate-data.yml) and is also chained into -the heavier TechEngine validation workflow as a downstream job. +## Local workflow ```bash python -m app.validate +cd site && npm ci && npm run build ``` -The validator uses only the Python standard library; no install step required. - -## Contributing +Use TechEngine for ingest, API, and dump generation. Do not change its code or +submodule pointer as part of a data-only change. The committed +`site/public/v1/` dump is deployed by GitHub Pages; its contents must remain in +the repository even when it is too large to browse comfortably locally. -Open a PR with the new/updated JSON file. The PR template walks through what to -include. The validator must pass. All records (`brand`, `soc`, `smartphone`, -`gpu`, and `cpu`) must include `source_urls` with at least one canonical -reference (vendor product page, Wikipedia infobox, datasheet). +For a focused workflow and the boundary between this repository and TechEngine, +see [docs/LOCAL_WORKFLOW.md](docs/LOCAL_WORKFLOW.md). ## License -Data is licensed **CC-BY-SA 4.0**; attribute "Data from TechAPI" and share alike. -The bundled validator code is [MIT](LICENSE). +Data is licensed **CC-BY-SA 4.0**. The bundled validator is [MIT](LICENSE). diff --git a/app/validate.py b/app/validate.py index b86983ea196e..e90d08fa03d7 100644 --- a/app/validate.py +++ b/app/validate.py @@ -1,110 +1,112 @@ -"""Validate seed JSON against the schema and conventions (§9.3, §15.3). - -Checks: required fields, slug convention (§14.1), value ranges/units (§14.3), -and foreign-key integrity by slug. Run with ``python -m scripts.validate``; -exits non-zero on the first failure set (used by CI ``validate-data.yml``). -""" - +"""Validate seed JSON against the schema and conventions (§9.3, §15.3). + +Checks: required fields, slug convention (§14.1), value ranges/units (§14.3), +and foreign-key integrity by slug. Run with ``python -m scripts.validate``; +exits non-zero on the first failure set (used by CI ``validate-data.yml``). +""" + from __future__ import annotations +import argparse import json -import re -import sys -from pathlib import Path -from typing import Any - -DATA_DIR = Path(__file__).resolve().parent.parent / "data" - -SLUG_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") - -BRAND_REQUIRED = {"slug", "name", "country", "categories", "source_urls"} -BRAND_CATEGORIES = { - "smartphone-oem", - "soc-designer", - "cpu-designer", - "gpu-designer", - "ip-licensor", - "aib-partner", - "pc-oem", - "chipset-maker", - "sub-brand", - "defunct", -} -COUNTRY_RE = re.compile(r"^[A-Z]{2}$") -SOC_REQUIRED = {"slug", "name", "manufacturer", "release_date", "process_nm", "gpu_name"} -PHONE_REQUIRED = { - "slug", - "name", - "brand", - "soc", - "release_date", - "ram_gb", - "battery_mah", - "weight_g", - "os", -} - -MOBILE_DEVICE_REQUIRED = { - "slug", - "name", - "brand", - "release_date", - "ram_gb", - "battery_mah", - "weight_g", - "os", - "source_urls", - "verified", -} - -GPU_REQUIRED = { - "slug", - "name", - "manufacturer", - "architecture", - "release_date", - "memory_gb", - "memory_type", - "memory_bus_bit", - "base_clock_mhz", - "boost_clock_mhz", - "tdp_w", - "pcie_version", -} - -CPU_REQUIRED = { - "slug", - "name", - "manufacturer", - "release_date", - "segment", - "architecture", - "cores", - "threads", -} - -LAPTOP_REQUIRED = { - "slug", - "name", - "brand", - "release_date", - "ram_gb", - "os", - "source_urls", - "verified", -} - -MONITOR_REQUIRED = { - "slug", - "name", - "brand", - "release_date", - "size_inch", - "resolution", - "source_urls", - "verified", -} - +import os +import re +import sys +from pathlib import Path +from typing import Any + +DATA_DIR = Path(__file__).resolve().parent.parent / "data" + +SLUG_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") + +BRAND_REQUIRED = {"slug", "name", "country", "categories", "source_urls"} +BRAND_CATEGORIES = { + "smartphone-oem", + "soc-designer", + "cpu-designer", + "gpu-designer", + "ip-licensor", + "aib-partner", + "pc-oem", + "chipset-maker", + "sub-brand", + "defunct", +} +COUNTRY_RE = re.compile(r"^[A-Z]{2}$") +SOC_REQUIRED = {"slug", "name", "manufacturer", "release_date", "process_nm", "gpu_name"} +PHONE_REQUIRED = { + "slug", + "name", + "brand", + "soc", + "release_date", + "ram_gb", + "battery_mah", + "weight_g", + "os", +} + +MOBILE_DEVICE_REQUIRED = { + "slug", + "name", + "brand", + "release_date", + "ram_gb", + "battery_mah", + "weight_g", + "os", + "source_urls", + "verified", +} + +GPU_REQUIRED = { + "slug", + "name", + "manufacturer", + "architecture", + "release_date", + "memory_gb", + "memory_type", + "memory_bus_bit", + "base_clock_mhz", + "boost_clock_mhz", + "tdp_w", + "pcie_version", +} + +CPU_REQUIRED = { + "slug", + "name", + "manufacturer", + "release_date", + "segment", + "architecture", + "cores", + "threads", +} + +LAPTOP_REQUIRED = { + "slug", + "name", + "brand", + "release_date", + "ram_gb", + "os", + "source_urls", + "verified", +} + +MONITOR_REQUIRED = { + "slug", + "name", + "brand", + "release_date", + "size_inch", + "resolution", + "source_urls", + "verified", +} + GAME_REQUIRED = { "slug", "name", @@ -112,344 +114,487 @@ "verified", } -SOFTWARE_REQUIRED = { - "slug", - "name", - "source_urls", - "verified", +GAME_LIST_FIELDS = ( + "platforms", + "genres", + "stores", + "developers", + "publishers", + "tags", +) + +GAME_ESRB_RATINGS = { + "Everyone", + "Everyone 10+", + "Teen", + "Mature", + "Adults Only", + "Rating Pending", } -DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") - - -def _load(subdir: str) -> list[tuple[str, dict[str, Any]]]: - path = DATA_DIR / subdir - if not path.exists(): - return [] - return [ - (str(f.relative_to(DATA_DIR)), json.loads(f.read_text(encoding="utf-8-sig"))) - for f in sorted(path.rglob("*.json")) # recurse into brand subfolders - ] - - -def _check_required( - name: str, record: dict[str, Any], required: set[str], errors: list[str] -) -> None: - missing = required - record.keys() - if missing: - errors.append(f"{name}: missing required fields {sorted(missing)}") - - -def _check_slug(name: str, slug: object, errors: list[str]) -> None: - if not isinstance(slug, str) or not SLUG_RE.match(slug): - errors.append(f"{name}: invalid slug '{slug}' (must be kebab-case, §14.1)") - - -def _check_range( - name: str, field: str, value: object, lo: float, hi: float, errors: list[str] -) -> None: - if value is None: - return - if not isinstance(value, (int, float)) or not (lo <= value <= hi): - errors.append(f"{name}: {field}={value} out of range [{lo}, {hi}]") - +def _game_shard(slug: str) -> str: + """Return the stable on-disk shard for a game slug.""" + return slug[:2] + +SOFTWARE_REQUIRED = { + "slug", + "name", + "source_urls", + "verified", +} + +DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +def _load(subdir: str) -> list[tuple[str, dict[str, Any]]]: + path = DATA_DIR / subdir + if not path.exists(): + return [] + return [ + (str(f.relative_to(DATA_DIR)), json.loads(f.read_text(encoding="utf-8-sig"))) + for f in sorted(path.rglob("*.json")) # recurse into brand subfolders + ] + + +def _check_required( + name: str, record: dict[str, Any], required: set[str], errors: list[str] +) -> None: + missing = required - record.keys() + if missing: + errors.append(f"{name}: missing required fields {sorted(missing)}") + + +def _check_slug(name: str, slug: object, errors: list[str]) -> None: + if not isinstance(slug, str) or not SLUG_RE.match(slug): + errors.append(f"{name}: invalid slug '{slug}' (must be kebab-case, §14.1)") + + +def _check_range( + name: str, field: str, value: object, lo: float, hi: float, errors: list[str] +) -> None: + if value is None: + return + if not isinstance(value, (int, float)) or not (lo <= value <= hi): + errors.append(f"{name}: {field}={value} out of range [{lo}, {hi}]") + + def _check_date(name: str, value: object, errors: list[str]) -> None: if not isinstance(value, str) or not DATE_RE.match(value): errors.append(f"{name}: release_date '{value}' must be ISO 8601 YYYY-MM-DD (§14.2)") -def _check_unique_slugs( - category: str, records: list[tuple[str, dict[str, Any]]], errors: list[str] -) -> None: - """Each category's `slug` must be unique — seed/dump load into a UNIQUE column.""" - seen: dict[str, str] = {} - for fname, rec in records: - slug = rec.get("slug") - if not isinstance(slug, str): - continue - if slug in seen: - errors.append( - f"{fname}: duplicate {category} slug '{slug}' (also in {seen[slug]})" - ) - else: - seen[slug] = fname +def _check_title(name: str, value: object, errors: list[str]) -> None: + if not isinstance(value, str) or not value.strip(): + errors.append(f"{name}: name must be a non-empty string") -def _check_source_urls(name: str, record: dict[str, Any], errors: list[str]) -> None: - urls = record.get("source_urls") - if not isinstance(urls, list) or not urls or not all( - isinstance(url, str) and url.startswith(("http://", "https://")) for url in urls - ): - errors.append(f"{name}: source_urls must be a non-empty list of http(s) URL strings") +def _check_bool(name: str, field: str, value: object, errors: list[str]) -> None: + if not isinstance(value, bool): + errors.append(f"{name}: {field} must be a boolean") -def _check_storage_options_gb(name: str, record: dict[str, Any], errors: list[str]) -> None: - values = record.get("storage_options_gb") - if values is None: +def _check_string_list( + name: str, + field: str, + value: object, + errors: list[str], + *, + allow_case_duplicates: bool = False, +) -> None: + """Validate optional game classification lists without constraining vocabulary.""" + if value is None: return - if not isinstance(values, list): - errors.append(f"{name}: storage_options_gb must be a list of integer GB values") + if not isinstance(value, list) or not all( + isinstance(item, str) and item.strip() for item in value + ): + errors.append(f"{name}: {field} must be a list of non-empty strings") return - bad = [value for value in values if not isinstance(value, int) or value < 1] - if bad: - errors.append(f"{name}: storage_options_gb contains invalid integer GB values {bad}") + if not allow_case_duplicates: + normalized = [item.casefold() for item in value] + if len(set(normalized)) != len(normalized): + errors.append(f"{name}: {field} contains duplicate values") +def _check_optional_url(name: str, field: str, value: object, errors: list[str]) -> None: + if value is not None and ( + not isinstance(value, str) or not value.startswith(("http://", "https://")) + ): + errors.append(f"{name}: {field} must be an http(s) URL when present") + + +def _check_unique_slugs( + category: str, records: list[tuple[str, dict[str, Any]]], errors: list[str] +) -> None: + """Each category's `slug` must be unique — seed/dump load into a UNIQUE column.""" + seen: dict[str, str] = {} + for fname, rec in records: + slug = rec.get("slug") + if not isinstance(slug, str): + continue + if slug in seen: + errors.append( + f"{fname}: duplicate {category} slug '{slug}' (also in {seen[slug]})" + ) + else: + seen[slug] = fname + + +def _check_source_urls(name: str, record: dict[str, Any], errors: list[str]) -> None: + urls = record.get("source_urls") + if not isinstance(urls, list) or not urls or not all( + isinstance(url, str) and url.startswith(("http://", "https://")) for url in urls + ): + errors.append(f"{name}: source_urls must be a non-empty list of http(s) URL strings") + + +def _check_storage_options_gb(name: str, record: dict[str, Any], errors: list[str]) -> None: + values = record.get("storage_options_gb") + if values is None: + return + if not isinstance(values, list): + errors.append(f"{name}: storage_options_gb must be a list of integer GB values") + return + bad = [value for value in values if not isinstance(value, int) or value < 1] + if bad: + errors.append(f"{name}: storage_options_gb contains invalid integer GB values {bad}") + + def _check_variant_path( - fname: str, - rec: dict[str, Any], - category: str, - errors: list[str], - *, - allow_flat: bool = False, -) -> None: + fname: str, + rec: dict[str, Any], + category: str, + errors: list[str], + *, + allow_flat: bool = False, +) -> None: + parts = Path(fname).parts + if allow_flat and len(parts) == 4: + return + if len(parts) != 5: + errors.append( + f"{fname}: {category} variants must live at " + f"'{category}////.json'" + ) + return + _, brand, year, base_model_slug, filename = parts + if rec.get("brand") != brand: + errors.append(f"{fname}: lives in brand '{brand}' but brand='{rec.get('brand')}'") + release_year = str(rec.get("release_date", ""))[:4] + if release_year and year != release_year: + errors.append( + f"{fname}: lives in year '{year}' but release_date starts with '{release_year}'" + ) + if rec.get("base_model_slug") and rec.get("base_model_slug") != base_model_slug: + errors.append( + f"{fname}: lives under base '{base_model_slug}' but " + f"base_model_slug='{rec.get('base_model_slug')}'" + ) + if filename != f"{rec.get('slug')}.json": + errors.append(f"{fname}: filename must match slug '{rec.get('slug')}'") + + +def _check_game_path(fname: str, rec: dict[str, Any], errors: list[str]) -> None: + """Keep imports in the documented ``game//.json`` layout.""" parts = Path(fname).parts - if allow_flat and len(parts) == 4: + slug = rec.get("slug") + if not isinstance(slug, str): return - if len(parts) != 5: - errors.append( - f"{fname}: {category} variants must live at " - f"'{category}////.json'" - ) + if len(parts) != 3: + errors.append(f"{fname}: game records must live at 'game//.json'") return - _, brand, year, base_model_slug, filename = parts - if rec.get("brand") != brand: - errors.append(f"{fname}: lives in brand '{brand}' but brand='{rec.get('brand')}'") - release_year = str(rec.get("release_date", ""))[:4] - if release_year and year != release_year: + _, shard, filename = parts + expected_shard = _game_shard(slug) + if shard != expected_shard: errors.append( - f"{fname}: lives in year '{year}' but release_date starts with '{release_year}'" + f"{fname}: lives in shard '{shard}' but slug '{slug}' belongs in '{expected_shard}'" ) - if rec.get("base_model_slug") and rec.get("base_model_slug") != base_model_slug: - errors.append( - f"{fname}: lives under base '{base_model_slug}' but " - f"base_model_slug='{rec.get('base_model_slug')}'" + if filename != f"{slug}.json": + errors.append(f"{fname}: filename must match slug '{slug}'") + + +def _validate_game_record(fname: str, rec: dict[str, Any], errors: list[str]) -> None: + _check_required(fname, rec, GAME_REQUIRED, errors) + _check_source_urls(fname, rec, errors) + _check_slug(fname, rec.get("slug"), errors) + _check_title(fname, rec.get("name"), errors) + _check_bool(fname, "verified", rec.get("verified"), errors) + if rec.get("release_date") is not None: + _check_date(fname, rec["release_date"], errors) + if rec.get("rating") is not None: + _check_range(fname, "rating", rec.get("rating"), 0, 5, errors) + if rec.get("rating_count") is not None: + _check_range(fname, "rating_count", rec.get("rating_count"), 0, 2_000_000_000, errors) + if rec.get("metacritic") is not None: + _check_range(fname, "metacritic", rec.get("metacritic"), 0, 100, errors) + if rec.get("playtime_hours") is not None: + _check_range(fname, "playtime_hours", rec.get("playtime_hours"), 0, 100_000, errors) + for field in GAME_LIST_FIELDS: + _check_string_list( + fname, + field, + rec.get(field), + errors, + allow_case_duplicates=field in {"developers", "publishers"}, ) - if filename != f"{rec.get('slug')}.json": - errors.append(f"{fname}: filename must match slug '{rec.get('slug')}'") + esrb_rating = rec.get("esrb_rating") + if esrb_rating is not None and esrb_rating not in GAME_ESRB_RATINGS: + errors.append(f"{fname}: esrb_rating '{esrb_rating}' not in {sorted(GAME_ESRB_RATINGS)}") + _check_optional_url(fname, "background_image", rec.get("background_image"), errors) + _check_game_path(fname, rec, errors) -def validate() -> list[str]: +def validate_games(data_dir: Path | None = None) -> list[str]: + """Stream game records so the import-scale dataset is not retained in memory.""" errors: list[str] = [] + root = (data_dir or DATA_DIR) / "game" + if not root.exists(): + return errors - brands = _load("brand") - socs = _load("soc") - phones = _load("smartphone") - tablets = _load("tablet") - watches = _load("watch") - pdas = _load("pda") - gpus = _load("gpu") - cpus = _load("cpu") - laptops = _load("laptop") - monitors = _load("monitor") - games = _load("game") + seen: dict[str, str] = {} + for directory, directories, filenames in os.walk(root): + directories.sort() + for filename in sorted(filenames): + if not filename.endswith(".json"): + continue + path = Path(directory) / filename + fname = str(path.relative_to(data_dir or DATA_DIR)) + try: + rec = json.loads(path.read_text(encoding="utf-8-sig")) + except (OSError, json.JSONDecodeError) as exc: + errors.append(f"{fname}: invalid JSON ({exc})") + continue + if not isinstance(rec, dict): + errors.append(f"{fname}: game record must be a JSON object") + continue + slug = rec.get("slug") + if isinstance(slug, str): + if slug in seen: + errors.append(f"{fname}: duplicate game slug '{slug}' (also in {seen[slug]})") + else: + seen[slug] = fname + _validate_game_record(fname, rec, errors) + return errors + + +def validate() -> list[str]: + errors: list[str] = [] + + brands = _load("brand") + socs = _load("soc") + phones = _load("smartphone") + tablets = _load("tablet") + watches = _load("watch") + pdas = _load("pda") + gpus = _load("gpu") + cpus = _load("cpu") + laptops = _load("laptop") + monitors = _load("monitor") software = _load("software") - - brand_slugs = {rec["slug"] for _, rec in brands if "slug" in rec} - soc_slugs = {rec["slug"] for _, rec in socs if "slug" in rec} - cpu_slugs = {rec["slug"] for _, rec in cpus if "slug" in rec} - gpu_slugs = {rec["slug"] for _, rec in gpus if "slug" in rec} - - for category, records in ( - ("brand", brands), - ("soc", socs), - ("smartphone", phones), - ("tablet", tablets), - ("watch", watches), - ("pda", pdas), - ("gpu", gpus), - ("cpu", cpus), - ("laptop", laptops), - ("monitor", monitors), - ("game", games), + + brand_slugs = {rec["slug"] for _, rec in brands if "slug" in rec} + soc_slugs = {rec["slug"] for _, rec in socs if "slug" in rec} + cpu_slugs = {rec["slug"] for _, rec in cpus if "slug" in rec} + gpu_slugs = {rec["slug"] for _, rec in gpus if "slug" in rec} + + for category, records in ( + ("brand", brands), + ("soc", socs), + ("smartphone", phones), + ("tablet", tablets), + ("watch", watches), + ("pda", pdas), + ("gpu", gpus), + ("cpu", cpus), + ("laptop", laptops), + ("monitor", monitors), ("software", software), - ): - _check_unique_slugs(category, records, errors) - - for fname, rec in brands: - _check_required(fname, rec, BRAND_REQUIRED, errors) - _check_source_urls(fname, rec, errors) - _check_slug(fname, rec.get("slug"), errors) - if "founded_year" in rec: - _check_range(fname, "founded_year", rec["founded_year"], 1800, 2100, errors) - country = rec.get("country") - if country is not None and not (isinstance(country, str) and COUNTRY_RE.match(country)): - errors.append(f"{fname}: country '{country}' must be ISO 3166 alpha-2 (e.g. 'KR')") - cats = rec.get("categories") - if not isinstance(cats, list) or not cats: - errors.append(f"{fname}: categories must be a non-empty list") - else: - bad = [c for c in cats if c not in BRAND_CATEGORIES] - if bad: - errors.append( - f"{fname}: invalid categories {bad}; allowed = {sorted(BRAND_CATEGORIES)}" - ) - if len(set(cats)) != len(cats): - errors.append(f"{fname}: categories contains duplicates") - # Path convention: brand//.json - parts = Path(fname).parts - if len(parts) != 3: - errors.append( - f"{fname}: must live at 'brand//.json' " - f"(got {len(parts) - 1} subpath components)" - ) - elif isinstance(country, str) and parts[1] != country.lower(): - errors.append( - f"{fname}: lives in '{parts[1]}/' but country='{country}' " - f"(expected '{country.lower()}/')" - ) - - for fname, rec in socs: - _check_required(fname, rec, SOC_REQUIRED, errors) - _check_source_urls(fname, rec, errors) - _check_slug(fname, rec.get("slug"), errors) - if "release_date" in rec: - _check_date(fname, rec["release_date"], errors) - _check_range(fname, "process_nm", rec.get("process_nm"), 1.0, 100.0, errors) - if rec.get("manufacturer") not in brand_slugs: - errors.append(f"{fname}: manufacturer '{rec.get('manufacturer')}' not a known brand") - - for fname, rec in phones: - _check_required(fname, rec, PHONE_REQUIRED, errors) - _check_source_urls(fname, rec, errors) - _check_slug(fname, rec.get("slug"), errors) - if "release_date" in rec: - _check_date(fname, rec["release_date"], errors) - _check_range(fname, "ram_gb", rec.get("ram_gb"), 1, 64, errors) - _check_range(fname, "battery_mah", rec.get("battery_mah"), 500, 12000, errors) - _check_range(fname, "weight_g", rec.get("weight_g"), 50, 500, errors) - if "msrp_usd" in rec: - _check_range(fname, "msrp_usd", rec["msrp_usd"], 50, 5000, errors) - if rec.get("brand") not in brand_slugs: - errors.append(f"{fname}: brand '{rec.get('brand')}' not a known brand") - if rec.get("soc") not in soc_slugs: - errors.append(f"{fname}: soc '{rec.get('soc')}' not a known SoC") - _check_variant_path(fname, rec, "smartphone", errors, allow_flat=True) - - for category, records in (("tablet", tablets), ("watch", watches), ("pda", pdas)): - for fname, rec in records: - _check_required(fname, rec, MOBILE_DEVICE_REQUIRED, errors) - _check_source_urls(fname, rec, errors) - _check_slug(fname, rec.get("slug"), errors) - if "release_date" in rec: - _check_date(fname, rec["release_date"], errors) - _check_range(fname, "ram_gb", rec.get("ram_gb"), 0.016, 64, errors) - _check_range(fname, "battery_mah", rec.get("battery_mah"), 50, 20000, errors) - _check_range(fname, "weight_g", rec.get("weight_g"), 10, 2000, errors) - if "msrp_usd" in rec: - _check_range(fname, "msrp_usd", rec["msrp_usd"], 10, 10000, errors) - if rec.get("brand") not in brand_slugs: - errors.append(f"{fname}: brand '{rec.get('brand')}' not a known brand") - if rec.get("soc") is not None and rec.get("soc") not in soc_slugs: - errors.append(f"{fname}: soc '{rec.get('soc')}' not a known SoC") - _check_storage_options_gb(fname, rec, errors) - _check_variant_path(fname, rec, category, errors) - - for fname, rec in gpus: - _check_required(fname, rec, GPU_REQUIRED, errors) - _check_source_urls(fname, rec, errors) - _check_slug(fname, rec.get("slug"), errors) - if "release_date" in rec: - _check_date(fname, rec["release_date"], errors) - _check_range(fname, "memory_gb", rec.get("memory_gb"), 0.001, 512, errors) - _check_range(fname, "tdp_w", rec.get("tdp_w"), 1, 3000, errors) - if "msrp_usd" in rec: - _check_range(fname, "msrp_usd", rec["msrp_usd"], 50, 100000, errors) - if rec.get("manufacturer") not in brand_slugs: - errors.append(f"{fname}: manufacturer '{rec.get('manufacturer')}' not a known brand") - - valid_segments = {"desktop", "laptop", "hedt", "server"} - for fname, rec in cpus: - _check_required(fname, rec, CPU_REQUIRED, errors) - _check_source_urls(fname, rec, errors) - _check_slug(fname, rec.get("slug"), errors) - if "release_date" in rec: - _check_date(fname, rec["release_date"], errors) - _check_range(fname, "cores", rec.get("cores"), 1, 512, errors) - _check_range(fname, "threads", rec.get("threads"), 1, 1024, errors) - if "msrp_usd" in rec: - _check_range(fname, "msrp_usd", rec["msrp_usd"], 20, 50000, errors) - if rec.get("segment") not in valid_segments: - seg = rec.get("segment") - errors.append(f"{fname}: segment '{seg}' not in {sorted(valid_segments)}") - if rec.get("manufacturer") not in brand_slugs: - errors.append(f"{fname}: manufacturer '{rec.get('manufacturer')}' not a known brand") - - for fname, rec in laptops: - _check_required(fname, rec, LAPTOP_REQUIRED, errors) - _check_source_urls(fname, rec, errors) - _check_slug(fname, rec.get("slug"), errors) - if "release_date" in rec: - _check_date(fname, rec["release_date"], errors) - _check_range(fname, "ram_gb", rec.get("ram_gb"), 1, 256, errors) - if rec.get("storage_gb") is not None: - _check_range(fname, "storage_gb", rec.get("storage_gb"), 1, 65536, errors) - if rec.get("weight_g") is not None: - _check_range(fname, "weight_g", rec.get("weight_g"), 300, 6000, errors) - if "msrp_usd" in rec: - _check_range(fname, "msrp_usd", rec["msrp_usd"], 50, 50000, errors) - if rec.get("brand") not in brand_slugs: - errors.append(f"{fname}: brand '{rec.get('brand')}' not a known brand") - if rec.get("cpu") is not None and rec.get("cpu") not in cpu_slugs: - errors.append(f"{fname}: cpu '{rec.get('cpu')}' not a known CPU") - if rec.get("gpu") is not None and rec.get("gpu") not in gpu_slugs: - errors.append(f"{fname}: gpu '{rec.get('gpu')}' not a known GPU") - _check_variant_path(fname, rec, "laptop", errors, allow_flat=True) - - for fname, rec in monitors: - _check_required(fname, rec, MONITOR_REQUIRED, errors) - _check_source_urls(fname, rec, errors) - _check_slug(fname, rec.get("slug"), errors) - if "release_date" in rec: - _check_date(fname, rec["release_date"], errors) - _check_range(fname, "size_inch", rec.get("size_inch"), 5, 120, errors) - _check_range(fname, "refresh_hz", rec.get("refresh_hz"), 24, 1000, errors) - if rec.get("ppi") is not None: - _check_range(fname, "ppi", rec.get("ppi"), 20, 1000, errors) - if rec.get("rating") is not None: - _check_range(fname, "rating", rec.get("rating"), 0, 5, errors) - if "msrp_usd" in rec: - _check_range(fname, "msrp_usd", rec["msrp_usd"], 10, 50000, errors) - if rec.get("brand") not in brand_slugs: - errors.append(f"{fname}: brand '{rec.get('brand')}' not a known brand") - _check_variant_path(fname, rec, "monitor", errors, allow_flat=True) - - for fname, rec in games: - _check_required(fname, rec, GAME_REQUIRED, errors) - _check_source_urls(fname, rec, errors) - _check_slug(fname, rec.get("slug"), errors) - if rec.get("release_date") is not None: - _check_date(fname, rec["release_date"], errors) - if rec.get("rating") is not None: - _check_range(fname, "rating", rec.get("rating"), 0, 5, errors) - if rec.get("metacritic") is not None: - _check_range(fname, "metacritic", rec.get("metacritic"), 0, 100, errors) - + ): + _check_unique_slugs(category, records, errors) + + for fname, rec in brands: + _check_required(fname, rec, BRAND_REQUIRED, errors) + _check_source_urls(fname, rec, errors) + _check_slug(fname, rec.get("slug"), errors) + if "founded_year" in rec: + _check_range(fname, "founded_year", rec["founded_year"], 1800, 2100, errors) + country = rec.get("country") + if country is not None and not (isinstance(country, str) and COUNTRY_RE.match(country)): + errors.append(f"{fname}: country '{country}' must be ISO 3166 alpha-2 (e.g. 'KR')") + cats = rec.get("categories") + if not isinstance(cats, list) or not cats: + errors.append(f"{fname}: categories must be a non-empty list") + else: + bad = [c for c in cats if c not in BRAND_CATEGORIES] + if bad: + errors.append( + f"{fname}: invalid categories {bad}; allowed = {sorted(BRAND_CATEGORIES)}" + ) + if len(set(cats)) != len(cats): + errors.append(f"{fname}: categories contains duplicates") + # Path convention: brand//.json + parts = Path(fname).parts + if len(parts) != 3: + errors.append( + f"{fname}: must live at 'brand//.json' " + f"(got {len(parts) - 1} subpath components)" + ) + elif isinstance(country, str) and parts[1] != country.lower(): + errors.append( + f"{fname}: lives in '{parts[1]}/' but country='{country}' " + f"(expected '{country.lower()}/')" + ) + + for fname, rec in socs: + _check_required(fname, rec, SOC_REQUIRED, errors) + _check_source_urls(fname, rec, errors) + _check_slug(fname, rec.get("slug"), errors) + if "release_date" in rec: + _check_date(fname, rec["release_date"], errors) + _check_range(fname, "process_nm", rec.get("process_nm"), 1.0, 100.0, errors) + if rec.get("manufacturer") not in brand_slugs: + errors.append(f"{fname}: manufacturer '{rec.get('manufacturer')}' not a known brand") + + for fname, rec in phones: + _check_required(fname, rec, PHONE_REQUIRED, errors) + _check_source_urls(fname, rec, errors) + _check_slug(fname, rec.get("slug"), errors) + if "release_date" in rec: + _check_date(fname, rec["release_date"], errors) + _check_range(fname, "ram_gb", rec.get("ram_gb"), 1, 64, errors) + _check_range(fname, "battery_mah", rec.get("battery_mah"), 500, 12000, errors) + _check_range(fname, "weight_g", rec.get("weight_g"), 50, 500, errors) + if "msrp_usd" in rec: + _check_range(fname, "msrp_usd", rec["msrp_usd"], 50, 5000, errors) + if rec.get("brand") not in brand_slugs: + errors.append(f"{fname}: brand '{rec.get('brand')}' not a known brand") + if rec.get("soc") not in soc_slugs: + errors.append(f"{fname}: soc '{rec.get('soc')}' not a known SoC") + _check_variant_path(fname, rec, "smartphone", errors, allow_flat=True) + + for category, records in (("tablet", tablets), ("watch", watches), ("pda", pdas)): + for fname, rec in records: + _check_required(fname, rec, MOBILE_DEVICE_REQUIRED, errors) + _check_source_urls(fname, rec, errors) + _check_slug(fname, rec.get("slug"), errors) + if "release_date" in rec: + _check_date(fname, rec["release_date"], errors) + _check_range(fname, "ram_gb", rec.get("ram_gb"), 0.016, 64, errors) + _check_range(fname, "battery_mah", rec.get("battery_mah"), 50, 20000, errors) + _check_range(fname, "weight_g", rec.get("weight_g"), 10, 2000, errors) + if "msrp_usd" in rec: + _check_range(fname, "msrp_usd", rec["msrp_usd"], 10, 10000, errors) + if rec.get("brand") not in brand_slugs: + errors.append(f"{fname}: brand '{rec.get('brand')}' not a known brand") + if rec.get("soc") is not None and rec.get("soc") not in soc_slugs: + errors.append(f"{fname}: soc '{rec.get('soc')}' not a known SoC") + _check_storage_options_gb(fname, rec, errors) + _check_variant_path(fname, rec, category, errors) + + for fname, rec in gpus: + _check_required(fname, rec, GPU_REQUIRED, errors) + _check_source_urls(fname, rec, errors) + _check_slug(fname, rec.get("slug"), errors) + if "release_date" in rec: + _check_date(fname, rec["release_date"], errors) + _check_range(fname, "memory_gb", rec.get("memory_gb"), 0.001, 512, errors) + _check_range(fname, "tdp_w", rec.get("tdp_w"), 1, 3000, errors) + if "msrp_usd" in rec: + _check_range(fname, "msrp_usd", rec["msrp_usd"], 50, 100000, errors) + if rec.get("manufacturer") not in brand_slugs: + errors.append(f"{fname}: manufacturer '{rec.get('manufacturer')}' not a known brand") + + valid_segments = {"desktop", "laptop", "hedt", "server"} + for fname, rec in cpus: + _check_required(fname, rec, CPU_REQUIRED, errors) + _check_source_urls(fname, rec, errors) + _check_slug(fname, rec.get("slug"), errors) + if "release_date" in rec: + _check_date(fname, rec["release_date"], errors) + _check_range(fname, "cores", rec.get("cores"), 1, 512, errors) + _check_range(fname, "threads", rec.get("threads"), 1, 1024, errors) + if "msrp_usd" in rec: + _check_range(fname, "msrp_usd", rec["msrp_usd"], 20, 50000, errors) + if rec.get("segment") not in valid_segments: + seg = rec.get("segment") + errors.append(f"{fname}: segment '{seg}' not in {sorted(valid_segments)}") + if rec.get("manufacturer") not in brand_slugs: + errors.append(f"{fname}: manufacturer '{rec.get('manufacturer')}' not a known brand") + + for fname, rec in laptops: + _check_required(fname, rec, LAPTOP_REQUIRED, errors) + _check_source_urls(fname, rec, errors) + _check_slug(fname, rec.get("slug"), errors) + if "release_date" in rec: + _check_date(fname, rec["release_date"], errors) + _check_range(fname, "ram_gb", rec.get("ram_gb"), 1, 256, errors) + if rec.get("storage_gb") is not None: + _check_range(fname, "storage_gb", rec.get("storage_gb"), 1, 65536, errors) + if rec.get("weight_g") is not None: + _check_range(fname, "weight_g", rec.get("weight_g"), 300, 6000, errors) + if "msrp_usd" in rec: + _check_range(fname, "msrp_usd", rec["msrp_usd"], 50, 50000, errors) + if rec.get("brand") not in brand_slugs: + errors.append(f"{fname}: brand '{rec.get('brand')}' not a known brand") + if rec.get("cpu") is not None and rec.get("cpu") not in cpu_slugs: + errors.append(f"{fname}: cpu '{rec.get('cpu')}' not a known CPU") + if rec.get("gpu") is not None and rec.get("gpu") not in gpu_slugs: + errors.append(f"{fname}: gpu '{rec.get('gpu')}' not a known GPU") + _check_variant_path(fname, rec, "laptop", errors, allow_flat=True) + + for fname, rec in monitors: + _check_required(fname, rec, MONITOR_REQUIRED, errors) + _check_source_urls(fname, rec, errors) + _check_slug(fname, rec.get("slug"), errors) + if "release_date" in rec: + _check_date(fname, rec["release_date"], errors) + _check_range(fname, "size_inch", rec.get("size_inch"), 5, 120, errors) + _check_range(fname, "refresh_hz", rec.get("refresh_hz"), 24, 1000, errors) + if rec.get("ppi") is not None: + _check_range(fname, "ppi", rec.get("ppi"), 20, 1000, errors) + if rec.get("rating") is not None: + _check_range(fname, "rating", rec.get("rating"), 0, 5, errors) + if "msrp_usd" in rec: + _check_range(fname, "msrp_usd", rec["msrp_usd"], 10, 50000, errors) + if rec.get("brand") not in brand_slugs: + errors.append(f"{fname}: brand '{rec.get('brand')}' not a known brand") + _check_variant_path(fname, rec, "monitor", errors, allow_flat=True) + for fname, rec in software: - _check_required(fname, rec, SOFTWARE_REQUIRED, errors) - _check_source_urls(fname, rec, errors) - _check_slug(fname, rec.get("slug"), errors) - if rec.get("release_date") is not None: - _check_date(fname, rec["release_date"], errors) - + _check_required(fname, rec, SOFTWARE_REQUIRED, errors) + _check_source_urls(fname, rec, errors) + _check_slug(fname, rec.get("slug"), errors) + if rec.get("release_date") is not None: + _check_date(fname, rec["release_date"], errors) + + # Game data is import-scale. Keep the default validator bounded by + # streaming game records after the smaller cross-category checks. + errors.extend(validate_games()) return errors - - -def run() -> int: + + +def run(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Validate TechAPI record data") + parser.add_argument( + "--category", + choices=("all", "game"), + default="all", + help="validate all records (default) or stream the game dataset only", + ) + args = parser.parse_args(argv) try: sys.stdout.reconfigure(encoding="utf-8") # type: ignore[union-attr] except Exception: pass - errors = validate() - if errors: - print(f"❌ Data validation failed ({len(errors)} issue(s)):") - for err in errors: - print(f" - {err}") - return 1 - print("✅ Data validation passed") - return 0 - - -if __name__ == "__main__": - sys.exit(run()) + errors = validate_games() if args.category == "game" else validate() + if errors: + print(f"❌ Data validation failed ({len(errors)} issue(s)):") + for err in errors: + print(f" - {err}") + return 1 + print("✅ Data validation passed") + return 0 + + +if __name__ == "__main__": + sys.exit(run()) diff --git a/docs/DATA_LAYOUT.md b/docs/DATA_LAYOUT.md new file mode 100644 index 000000000000..73dc8c2d27f9 --- /dev/null +++ b/docs/DATA_LAYOUT.md @@ -0,0 +1,44 @@ +# Data layout + +Versioned records stay in `data/`; the directory name is the record family and +the JSON file is the individual record. Do not group records by a transient +import run or by a scraper name. + +## Games + +Game records use a deterministic shard: + +``` +data/game//.json +``` + +For example, `hades` belongs at `data/game/ha/hades.json`. This keeps a +large import stable and prevents one directory from becoming a bottleneck. + +## Local checkout policy + +The default checkout deliberately excludes `data/` and generated `site/` +content. They remain tracked and are still present on the remote; omitting +them only prevents routine Git polling from scanning millions of files. + +For a focused data import, temporarily add only the family you need: + +``` +git sparse-checkout add data/cpu +``` + +For a full game import, add `data/game` for the duration of the import, then +return to the small default checkout: + +``` +git sparse-checkout add data/game +git sparse-checkout set --cone .github app docs TechEngine +``` + +Never use `assume-unchanged` or `skip-worktree` as an import workflow. Run +`python -m app.validate` only while the needed data family is checked out. +For the million-record game import, use the streaming validator: + +``` +python -m app.validate --category game +``` diff --git a/docs/LOCAL_WORKFLOW.md b/docs/LOCAL_WORKFLOW.md new file mode 100644 index 000000000000..6bd0139c05a4 --- /dev/null +++ b/docs/LOCAL_WORKFLOW.md @@ -0,0 +1,53 @@ +# Local workflow + +## Ownership boundary + +TechAPI owns versioned record files, the small standard-library validator, and +the Astro Pages site. TechEngine owns ingestion, the API service, schemas, +database models, dump generation, and heavy validation. Keep a schema change +coordinated across both repositories; do not modify the `TechEngine` submodule +pointer for an ordinary data cleanup. + +## Game records + +Games are individual JSON files under `data/game//`. The shard is the +first two characters of `slug` (or the whole slug when it has one character). +The layout is intentionally independent of genre, platform, developer, and +import source so that imports stay stable and directories stay bounded. + +Before an import: + +1. Put each record in its slug shard. +2. Keep `slug`, `name`, `source_urls`, and `verified` present. +3. Run `python -m app.validate`. +4. Generate or verify the static dump through TechEngine when the API output + needs to change. + +## Generated files + +- `site/public/v1/` is committed API output used by GitHub Pages; do not delete + it as a local cleanup shortcut. +- `site/public/v1/history.json` and `verification.json` are build outputs and + are ignored. +- `.claude/archive/` contains local historical work notes and is ignored. +- Python caches (`.pytest_cache`, `.mypy_cache`, `.ruff_cache`) are disposable. + +## Large-repository guardrail + +The default sparse checkout intentionally leaves `data/` and `site/` out of +the working tree. Do not copy a large shard or generated dump back under this +repository just for local inspection: that makes Git expand the sparse index +on routine status checks. + +If Git reports that the sparse index is expanding, preserve the local artifact +in a sibling folder outside the repository, then reapply the documented +checkout: + +``` +git sparse-checkout reapply --sparse-index +git config core.fsmonitor true +git fsmonitor--daemon start +``` + +This keeps normal `git status` work proportional to changed files. Never use +`assume-unchanged` or `skip-worktree` to mask an import or generated dump. diff --git a/tests/test_game_validation.py b/tests/test_game_validation.py new file mode 100644 index 000000000000..6baf803b22f5 --- /dev/null +++ b/tests/test_game_validation.py @@ -0,0 +1,97 @@ +"""Focused checks for the game-record validation contract.""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from app import validate as validator + + +class GameValidationTests(unittest.TestCase): + def _validate_game(self, record: dict[str, object], *, shard: str = "ha") -> list[str]: + with tempfile.TemporaryDirectory() as temporary_directory: + data_dir = Path(temporary_directory) + record_path = data_dir / "game" / shard / f"{record['slug']}.json" + record_path.parent.mkdir(parents=True) + record_path.write_text(json.dumps(record), encoding="utf-8") + return validator.validate_games(data_dir) + + def test_accepts_well_formed_game_record(self) -> None: + errors = self._validate_game( + { + "slug": "hades", + "name": "Hades", + "verified": True, + "source_urls": ["https://example.com/hades"], + "release_date": "2020-09-17", + "rating": 4.8, + "rating_count": 1000, + "metacritic": 93, + "playtime_hours": 25, + "platforms": ["PC", "Nintendo Switch"], + "genres": ["Action"], + "developers": ["Supergiant Games"], + "esrb_rating": "Teen", + "background_image": "https://example.com/hades.jpg", + } + ) + self.assertEqual(errors, []) + + def test_rejects_invalid_game_metadata(self) -> None: + errors = self._validate_game( + { + "slug": "hades", + "name": " ", + "verified": "yes", + "source_urls": ["https://example.com/hades"], + "platforms": ["PC", 4], + "rating_count": -1, + "playtime_hours": -1, + "esrb_rating": "Unknown", + "background_image": "file:///tmp/hades.jpg", + } + ) + self.assertTrue(any("name must be a non-empty string" in error for error in errors)) + self.assertTrue(any("verified must be a boolean" in error for error in errors)) + self.assertTrue(any("platforms must be a list of non-empty strings" in error for error in errors)) + self.assertTrue(any("rating_count=-1 out of range" in error for error in errors)) + self.assertTrue(any("playtime_hours=-1 out of range" in error for error in errors)) + self.assertTrue(any("esrb_rating 'Unknown'" in error for error in errors)) + self.assertTrue(any("background_image must be an http(s) URL" in error for error in errors)) + + def test_rejects_game_outside_slug_shard(self) -> None: + errors = self._validate_game( + { + "slug": "hades", + "name": "Hades", + "verified": False, + "source_urls": ["https://example.com/hades"], + }, + shard="ga", + ) + self.assertTrue(any("belongs in 'ha'" in error for error in errors)) + + def test_default_validation_streams_games_instead_of_loading_them(self) -> None: + loaded_categories: list[str] = [] + + def load_without_games(category: str): + loaded_categories.append(category) + self.assertNotEqual(category, "game") + return [] + + with ( + mock.patch.object(validator, "_load", side_effect=load_without_games), + mock.patch.object(validator, "validate_games", return_value=[] ) as streamed_games, + ): + self.assertEqual(validator.validate(), []) + + self.assertNotIn("game", loaded_categories) + streamed_games.assert_called_once_with() + + +if __name__ == "__main__": + unittest.main()