diff --git a/CHANGELOG.md b/CHANGELOG.md index fe8bbdb..d88ff84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Change log +### 0.4.4 - 2026-07-01 +- Fixed dataset filename classification so names containing dataset tokens in the prefix, such as `River._edges_.nodes.geojson`, are classified by their final dataset suffix instead of by loose substring matching. +- Supported filename detection now consistently accepts only these four forms for each dataset type: `.geojson`, `*..geojson`, `.OSW.geojson`, and `*..OSW.geojson`. +- Rejected ambiguous glued or marker-style names such as `roadnodes.geojson`, `roadedges.geojson`, and `River._edges_.geojson` with clearer unsupported-file guidance. +- Added regression coverage using `tests/assets/3750.zip` plus unit tests for accepted and rejected filename formats across dataset keys. +- Updated the README supported-filenames section to document the accepted formats and invalid examples. + ### 0.4.3 - 2026-06-03 - Removed the `maximum: 5000` constraint from `length` in the OSW 0.3 edges and lines schemas so longer paths, including `length: 6629.35`, validate successfully. - Added regression coverage for long `length` values using `tests/assets/max-length-error.zip`, plus boundary tests for `length: 0` and `length: -1`. diff --git a/README.md b/README.md index a9a1f54..b0d2b48 100644 --- a/README.md +++ b/README.md @@ -72,29 +72,32 @@ validator = OSWValidation( ## Supported filenames -The validator accepts dataset files whose names end with one of these exact suffixes: +The validator accepts four filename formats for each dataset type: -- `.edges.geojson` -- `.lines.geojson` -- `.nodes.geojson` -- `.points.geojson` -- `.polygons.geojson` -- `.zones.geojson` +- `.geojson` +- `*..geojson` +- `.OSW.geojson` +- `*..OSW.geojson` -It also accepts the legacy form: +Where `` is one of: -- `.edges.OSW.geojson` -- `.lines.OSW.geojson` -- `.nodes.OSW.geojson` -- `.points.OSW.geojson` -- `.polygons.OSW.geojson` -- `.zones.OSW.geojson` +- `edges` +- `lines` +- `nodes` +- `points` +- `polygons` +- `zones` Examples: +- `nodes.geojson` is valid - `gs_metaline_falls_uga.nodes.geojson` is valid +- `nodes.OSW.geojson` is valid - `gs_yarrow_point.edges.geojson` is valid +- `wa.microsoft.graph.edges.OSW.geojson` is valid - `roadEdges.geojson` is invalid +- `roadedges.geojson` is invalid +- `River._edges_.geojson` is invalid If a dataset uses canonical OSW 0.3 names that start with `opensidewalks.`, then only these exact names are allowed: diff --git a/src/python_osw_validation/__init__.py b/src/python_osw_validation/__init__.py index 127e851..ac19ec3 100644 --- a/src/python_osw_validation/__init__.py +++ b/src/python_osw_validation/__init__.py @@ -8,7 +8,7 @@ import jsonschema_rs from .zipfile_handler import ZipFileHandler -from .extracted_data_validator import ExtractedDataValidator, OSW_DATASET_FILES +from .extracted_data_validator import ExtractedDataValidator, OSW_DATASET_FILES, dataset_key_for_filename from .version import __version__ from .helpers import ( _add_additional_properties_hint, @@ -269,18 +269,27 @@ def _schema_key_from_text(self, text: Optional[str]) -> Optional[str]: return None basename = os.path.basename(text).lower() - stem, _ = os.path.splitext(basename) for key in self.dataset_schema_paths: if ( - stem == key - or stem == f"{key}.osw" - or stem.endswith(f".{key}") - or stem.endswith(f".{key}.osw") + basename == f"{key}.geojson" + or basename == f"{key}.osw.geojson" + or basename.endswith(f".{key}.geojson") + or basename.endswith(f".{key}.osw.geojson") ): return key return None + def _schema_key_from_schema_url(self, schema_url: Any) -> Optional[str]: + if not isinstance(schema_url, str): + return None + + lower_url = schema_url.lower() + for key in self.dataset_schema_paths: + if lower_url.endswith(f"/{key}.schema.json"): + return key + return None + def _is_nullish_value(self, value: Any) -> bool: if value is None: return True @@ -371,6 +380,10 @@ def pick_schema_for_file(self, file_path: str, geojson_data: Dict[str, Any]) -> if schema_key and schema_key in self.dataset_schema_paths: return self.dataset_schema_paths[schema_key] + schema_key = self._schema_key_from_schema_url(geojson_data.get('$schema')) + if schema_key and schema_key in self.dataset_schema_paths: + return self.dataset_schema_paths[schema_key] + return self.line_schema_path # ---------------------------- @@ -423,8 +436,7 @@ def _finalize(is_valid: bool, errors: Optional[List[str]] = None) -> ValidationR # Load GeoDataFrames for integrity checks for file in validator.files: file_path = os.path.join(file) - osw_file = next((osw_key for osw_key in OSW_DATASET_FILES.keys() - if osw_key in os.path.basename(file_path)), '') + osw_file = dataset_key_for_filename(os.path.basename(file_path)) try: gdf = _read_geojson_without_ext(file_path) except Exception as e: diff --git a/src/python_osw_validation/extracted_data_validator.py b/src/python_osw_validation/extracted_data_validator.py index a104a74..8dd0092 100644 --- a/src/python_osw_validation/extracted_data_validator.py +++ b/src/python_osw_validation/extracted_data_validator.py @@ -50,18 +50,21 @@ def _matches_dataset_filename(basename: str, dataset_key: str) -> bool: lower_name = basename.lower() - if not lower_name.endswith(".geojson"): - return False - - stem = lower_name[:-len(".geojson")] return ( - stem == dataset_key - or stem == f"{dataset_key}.osw" - or stem.endswith(f".{dataset_key}") - or stem.endswith(f".{dataset_key}.osw") + lower_name == f"{dataset_key}.geojson" + or lower_name == f"{dataset_key}.osw.geojson" + or lower_name.endswith(f".{dataset_key}.geojson") + or lower_name.endswith(f".{dataset_key}.osw.geojson") ) +def dataset_key_for_filename(basename: str): + for dataset_key in OSW_DATASET_FILES.keys(): + if _matches_dataset_filename(basename, dataset_key): + return dataset_key + return None + + class ExtractedDataValidator: def __init__(self, extracted_dir: str): self.extracted_dir = extracted_dir @@ -118,7 +121,12 @@ def is_valid(self) -> bool: ) if unsupported_files: allowed_fmt = ", ".join(allowed_keys) - allowed_names = f"*.{{{allowed_fmt}}}.geojson" + allowed_names = ( + f"{{{allowed_fmt}}}.geojson, " + f"*.{{{allowed_fmt}}}.geojson, " + f"{{{allowed_fmt}}}.OSW.geojson, or " + f"*.{{{allowed_fmt}}}.OSW.geojson" + ) self.error = (f"Unsupported .geojson files present: {', '.join(unsupported_files)}. " f"Allowed file names are {allowed_names}") return False diff --git a/src/python_osw_validation/version.py b/src/python_osw_validation/version.py index 908c0bb..9a8e054 100644 --- a/src/python_osw_validation/version.py +++ b/src/python_osw_validation/version.py @@ -1 +1 @@ -__version__ = '0.4.3' +__version__ = '0.4.4' diff --git a/tests/assets/3750.zip b/tests/assets/3750.zip new file mode 100644 index 0000000..fff232a Binary files /dev/null and b/tests/assets/3750.zip differ diff --git a/tests/unit_tests/test_extracted_data_validator.py b/tests/unit_tests/test_extracted_data_validator.py index 4eb1c1f..0fe0ecb 100644 --- a/tests/unit_tests/test_extracted_data_validator.py +++ b/tests/unit_tests/test_extracted_data_validator.py @@ -3,7 +3,7 @@ import tempfile import shutil from unittest.mock import patch -from src.python_osw_validation.extracted_data_validator import ExtractedDataValidator +from src.python_osw_validation.extracted_data_validator import ExtractedDataValidator, dataset_key_for_filename from src.python_osw_validation.extracted_data_validator import ALLOWED_OSW_03_FILENAMES @@ -75,6 +75,31 @@ def test_valid_legacy_osw_suffix_files(self): self.assertTrue(validator.is_valid()) self.assertEqual(len(validator.files), 2) + def test_valid_bare_dataset_filenames(self): + validator = ExtractedDataValidator(self.test_dir) + self.create_files(['nodes.geojson', 'edges.geojson']) + self.assertTrue(validator.is_valid()) + self.assertEqual(len(validator.files), 2) + + def test_valid_bare_legacy_osw_dataset_filenames(self): + validator = ExtractedDataValidator(self.test_dir) + self.create_files(['nodes.OSW.geojson', 'edges.OSW.geojson']) + self.assertTrue(validator.is_valid()) + self.assertEqual(len(validator.files), 2) + + def test_supported_legacy_filename_formats_for_all_dataset_keys(self): + for dataset_key in ('edges', 'nodes', 'points', 'lines', 'zones', 'polygons'): + self.assertEqual(dataset_key_for_filename(f'{dataset_key}.geojson'), dataset_key) + self.assertEqual(dataset_key_for_filename(f'city.{dataset_key}.geojson'), dataset_key) + self.assertEqual(dataset_key_for_filename(f'{dataset_key}.OSW.geojson'), dataset_key) + self.assertEqual(dataset_key_for_filename(f'city.{dataset_key}.OSW.geojson'), dataset_key) + + def test_ambiguous_dataset_name_substrings_are_not_supported(self): + self.assertIsNone(dataset_key_for_filename('roadnodes.geojson')) + self.assertIsNone(dataset_key_for_filename('roadedges.geojson')) + self.assertIsNone(dataset_key_for_filename('city._nodes_.geojson')) + self.assertIsNone(dataset_key_for_filename('River._edges_.geojson')) + def test_non_standard_filenames_raise_error(self): validator = ExtractedDataValidator(self.test_dir) self.create_files(['custom.nodes.geojson', 'opensidewalks.nodes.geojson']) @@ -112,7 +137,10 @@ def test_unsupported_files_are_rejected(self): self.assertEqual( validator.error, 'Unsupported .geojson files present: something_else.geojson. ' - 'Allowed file names are *.{edges, nodes, points, lines, zones, polygons}.geojson' + 'Allowed file names are {edges, nodes, points, lines, zones, polygons}.geojson, ' + '*.{edges, nodes, points, lines, zones, polygons}.geojson, ' + '{edges, nodes, points, lines, zones, polygons}.OSW.geojson, or ' + '*.{edges, nodes, points, lines, zones, polygons}.OSW.geojson' ) def test_glued_dataset_names_are_rejected(self): @@ -122,7 +150,10 @@ def test_glued_dataset_names_are_rejected(self): self.assertEqual( validator.error, 'Unsupported .geojson files present: roadedges.geojson, roadnodes.geojson. ' - 'Allowed file names are *.{edges, nodes, points, lines, zones, polygons}.geojson' + 'Allowed file names are {edges, nodes, points, lines, zones, polygons}.geojson, ' + '*.{edges, nodes, points, lines, zones, polygons}.geojson, ' + '{edges, nodes, points, lines, zones, polygons}.OSW.geojson, or ' + '*.{edges, nodes, points, lines, zones, polygons}.OSW.geojson' ) diff --git a/tests/unit_tests/test_osw_validation.py b/tests/unit_tests/test_osw_validation.py index aad0606..8fb4a9d 100644 --- a/tests/unit_tests/test_osw_validation.py +++ b/tests/unit_tests/test_osw_validation.py @@ -53,6 +53,7 @@ def setUp(self): self.edge_u_id_coord_mismatch = os.path.join(ASSETS_PATH, 'edge_u_id_coord_mismatch.zip') self.edge_v_id_coord_mismatch = os.path.join(ASSETS_PATH, 'edge_v_id_coord_mismatch.zip') self.zone_w_id_coord_mismatch = os.path.join(ASSETS_PATH, 'zone_w_id_coord_mismatch.zip') + self.edges_token_in_prefix_file = os.path.join(ASSETS_PATH, '3750.zip') def _validate_edge_length(self, length): data = { @@ -369,6 +370,14 @@ def test_geom_mapping_valid_passes(self): self.assertTrue(result.is_valid, f"Expected valid; errors={result.errors}") self.assertIsNone(result.errors) + def test_edges_token_in_filename_prefix_does_not_misclassify_nodes(self): + """A filename prefix containing '_edges_' should not make nodes load as edges.""" + validation = OSWValidation(zipfile_path=self.edges_token_in_prefix_file) + result = validation.validate() + + self.assertTrue(result.is_valid, f"Expected valid; errors={result.errors}") + self.assertIsNone(result.errors) + def test_edge_u_id_coord_mismatch_fails(self): """Edge whose start coordinate does not match the _u_id node is rejected.""" validation = OSWValidation(zipfile_path=self.edge_u_id_coord_mismatch) diff --git a/tests/unit_tests/test_osw_validation_extras.py b/tests/unit_tests/test_osw_validation_extras.py index f7f277d..3ccc9e9 100644 --- a/tests/unit_tests/test_osw_validation_extras.py +++ b/tests/unit_tests/test_osw_validation_extras.py @@ -915,7 +915,10 @@ def test_pick_schema_by_schema_url(self): def test_pick_schema_filename_fallback(self): v = OSWValidation(zipfile_path="dummy.zip") + self.assertEqual(v.pick_schema_for_file("/tmp/nodes.geojson", {"features": []}), v.dataset_schema_paths["nodes"]) self.assertEqual(v.pick_schema_for_file("/tmp/my.nodes.geojson", {"features": []}), v.dataset_schema_paths["nodes"]) + self.assertEqual(v.pick_schema_for_file("/tmp/nodes.OSW.geojson", {"features": []}), v.dataset_schema_paths["nodes"]) + self.assertEqual(v.pick_schema_for_file("/tmp/my.nodes.OSW.geojson", {"features": []}), v.dataset_schema_paths["nodes"]) self.assertEqual(v.pick_schema_for_file("/tmp/my.points.geojson", {"features": []}), v.dataset_schema_paths["points"]) self.assertEqual(v.pick_schema_for_file("/tmp/my.edges.geojson", {"features": []}), v.dataset_schema_paths["edges"]) self.assertEqual(v.pick_schema_for_file("/tmp/my.lines.geojson", {"features": []}), v.dataset_schema_paths["lines"]) @@ -948,6 +951,10 @@ def test_pick_schema_ignores_prefix_substrings(self): v.pick_schema_for_file("/tmp/roadEdges.geojson", {"features": []}), v.line_schema_path, ) + self.assertEqual( + v.pick_schema_for_file("/tmp/River._edges_.geojson", {"features": []}), + v.line_schema_path, + ) def test_pick_schema_force_single_schema_override(self): force = "/forced/opensidewalks.schema-0.3.json"