perf(validate): stream games in full validation

Author: Seungpyo1007

app/validate.py

@@ -390,8 +390,7 @@ def validate() -> list[str]:
     cpus = _load("cpu")
     laptops = _load("laptop")
     monitors = _load("monitor")
-    games = _load("game")
-    software = _load("software")
+    software = _load("software")
 
     brand_slugs = {rec["slug"] for _, rec in brands if "slug" in rec}
     soc_slugs = {rec["slug"] for _, rec in socs if "slug" in rec}
@@ -409,8 +408,7 @@ def validate() -> list[str]:
         ("cpu", cpus),
         ("laptop", laptops),
         ("monitor", monitors),
-        ("game", games),
-        ("software", software),
+        ("software", software),
     ):
         _check_unique_slugs(category, records, errors)
 
@@ -562,17 +560,17 @@ def validate() -> list[str]:
             errors.append(f"{fname}: brand '{rec.get('brand')}' not a known brand")
         _check_variant_path(fname, rec, "monitor", errors, allow_flat=True)
 
-    for fname, rec in games:
-        _validate_game_record(fname, rec, errors)
-
-    for fname, rec in software:
+    for fname, rec in software:
         _check_required(fname, rec, SOFTWARE_REQUIRED, errors)
         _check_source_urls(fname, rec, errors)
         _check_slug(fname, rec.get("slug"), errors)
         if rec.get("release_date") is not None:
             _check_date(fname, rec["release_date"], errors)
 
-    return errors
+    # Game data is import-scale. Keep the default validator bounded by
+    # streaming game records after the smaller cross-category checks.
+    errors.extend(validate_games())
+    return errors
 
 
 def run(argv: list[str] | None = None) -> int:

docs/DATA_LAYOUT.md

@@ -24,14 +24,14 @@ them only prevents routine Git polling from scanning millions of files.
 For a focused data import, temporarily add only the family you need:
 
 ```
-git sparse-checkout add --cone data/cpu
+git sparse-checkout add data/cpu
 ```
 
 For a full game import, add `data/game` for the duration of the import, then
 return to the small default checkout:
 
 ```
-git sparse-checkout add --cone data/game
+git sparse-checkout add data/game
 git sparse-checkout set --cone .github app docs TechEngine
 ```
 

tests/test_game_validation.py

@@ -6,6 +6,7 @@
 import tempfile
 import unittest
 from pathlib import Path
+from unittest import mock
 
 from app import validate as validator
 
@@ -47,7 +48,7 @@ def test_rejects_invalid_game_metadata(self) -> None:
                 "name": " ",
                 "verified": "yes",
                 "source_urls": ["https://example.com/hades"],
-                "platforms": ["PC", "pc"],
+                "platforms": ["PC", 4],
                 "rating_count": -1,
                 "playtime_hours": -1,
                 "esrb_rating": "Unknown",
@@ -56,7 +57,7 @@ def test_rejects_invalid_game_metadata(self) -> None:
         )
         self.assertTrue(any("name must be a non-empty string" in error for error in errors))
         self.assertTrue(any("verified must be a boolean" in error for error in errors))
-        self.assertTrue(any("platforms contains duplicate values" in error for error in errors))
+        self.assertTrue(any("platforms must be a list of non-empty strings" in error for error in errors))
         self.assertTrue(any("rating_count=-1 out of range" in error for error in errors))
         self.assertTrue(any("playtime_hours=-1 out of range" in error for error in errors))
         self.assertTrue(any("esrb_rating 'Unknown'" in error for error in errors))
@@ -74,6 +75,23 @@ def test_rejects_game_outside_slug_shard(self) -> None:
         )
         self.assertTrue(any("belongs in 'ha'" in error for error in errors))
 
+    def test_default_validation_streams_games_instead_of_loading_them(self) -> None:
+        loaded_categories: list[str] = []
+
+        def load_without_games(category: str):
+            loaded_categories.append(category)
+            self.assertNotEqual(category, "game")
+            return []
+
+        with (
+            mock.patch.object(validator, "_load", side_effect=load_without_games),
+            mock.patch.object(validator, "validate_games", return_value=[] ) as streamed_games,
+        ):
+            self.assertEqual(validator.validate(), [])
+
+        self.assertNotIn("game", loaded_categories)
+        streamed_games.assert_called_once_with()
+
 
 if __name__ == "__main__":
     unittest.main()