Fix Databricks tests for GEOGRAPHY and GEOMETRY columns

CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+- `datacontract import unity` no longer writes the `databricksType` custom property, which duplicated `physicalType`
+- `datacontract export sql --server databricks` keeps the declared length of `varchar(n)` and `char(n)` instead of exporting `STRING`
+
+### Fixed
+- `datacontract test` for Databricks no longer fails all checks of a model with a `GEOGRAPHY` or `GEOMETRY` column (#1483)
+
 ## [1.1.0] - 2026-08-04
 
 This release drops the pyspark compile-time dependency. The server types `dataframe` and `databricks` still work with a provided Spark session.

datacontract/engines/ibis/connections/connect.py

@@ -276,6 +276,12 @@ def _databricks_connect(ibis, **kwargs):
     """
     from ibis.backends.databricks import Backend
 
+    from datacontract.engines.ibis.connections.databricks_patch import apply_databricks_compatibility_patch
+
+    # Databricks-only column types (GEOGRAPHY(4326), …) otherwise fail the whole
+    # model when ibis reflects the table.
+    apply_databricks_compatibility_patch()
+
     original_post_connect = Backend._post_connect
     Backend._post_connect = lambda self, *, memtable_volume=None: None
     try:

datacontract/engines/ibis/connections/databricks_patch.py

@@ -0,0 +1,142 @@
+"""Make ibis's Databricks schema introspection survive Databricks-only column types.
+
+ibis reflects a Databricks table by reading ``DESCRIBE ... AS JSON`` and mapping
+every column through ``DatabricksType.from_string`` (see
+``ibis.backends.databricks._databricks_schema_to_ibis``). The whole model is
+reflected in one pass, so a single column ibis cannot represent raises out of
+``con.table(...)`` and every check of that model fails with the same error —
+including checks on unrelated columns.
+
+Two things are patched here:
+
+1. ``GEOGRAPHY`` / ``GEOMETRY`` with an SRID. ibis reads the first type
+   parameter as a geometry *subtype* (the PostGIS spelling
+   ``GEOGRAPHY(POINT, 4326)``), but Databricks declares only the SRID,
+   ``GEOGRAPHY(4326)``, so the subtype lookup fails with ``KeyError: '4326'``
+   (https://github.com/datacontract/datacontract-cli/issues/1483). A lone
+   numeric parameter is handed to ibis as the SRID instead.
+
+2. Anything else ibis still cannot convert becomes ``Unknown`` for that column
+   alone, with a warning naming it, so the rest of the model is still checked.
+   This mirrors what the pyspark path already does for Spark types ibis has no
+   mapping for (``_pyspark_table_unconvertible_as_unknown``).
+
+Both patches are process-global (ibis looks these up as class/module attributes)
+and idempotent. Point 1 can go once a fixed ibis release is available.
+"""
+
+from __future__ import annotations
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+_PATCHED_FLAG = "__datacontract_patched__"
+
+
+def apply_databricks_compatibility_patch() -> None:
+    """Patch ibis's Databricks type conversion. Safe to call more than once.
+
+    Best-effort: if the backend internals differ from what we expect (e.g. a
+    future ibis refactor), the patch is skipped and the originals stay in place
+    rather than breaking the connection.
+    """
+    try:
+        _patch_geospatial_srid()
+        _patch_unconvertible_columns()
+    except Exception:  # pragma: no cover - defensive, never block a connection
+        logger.debug("Could not apply Databricks compatibility patch", exc_info=True)
+
+
+def _patch_geospatial_srid() -> None:
+    """Read a lone numeric ``GEOGRAPHY`` / ``GEOMETRY`` parameter as the SRID."""
+    from ibis.backends.sql.datatypes import DatabricksType
+
+    for geotype in ("GEOGRAPHY", "GEOMETRY"):
+        original = getattr(DatabricksType, f"_from_sqlglot_{geotype}")
+        if getattr(original, _PATCHED_FLAG, False):
+            continue
+        setattr(DatabricksType, f"_from_sqlglot_{geotype}", _srid_aware(original))
+
+
+def _srid_aware(original):
+    """Wrap ibis's ``_from_sqlglot_GEO*`` to accept ``GEO*(<srid>)``.
+
+    ibis's signature is ``(subtype, srid)``; Databricks passes the SRID alone, so
+    a first parameter that is a plain number is moved into the ``srid`` slot. A
+    named subtype (``GEOMETRY(POINT, 4326)``) is left where it is.
+    """
+    # __func__ unwraps the classmethod so the wrapper can pass its own cls.
+    unbound = original.__func__
+
+    def from_sqlglot_geo(cls, arg=None, srid=None, nullable=None):
+        if srid is None and _is_numeric_param(arg):
+            arg, srid = None, arg
+        return unbound(cls, arg, srid, nullable=nullable)
+
+    from_sqlglot_geo.__name__ = unbound.__name__
+    setattr(from_sqlglot_geo, _PATCHED_FLAG, True)
+    return classmethod(from_sqlglot_geo)
+
+
+def _is_numeric_param(param) -> bool:
+    """True for a sqlglot type parameter that is a bare integer, e.g. ``4326``."""
+    if param is None:
+        return False
+    try:
+        int(param.this.this)
+    except (AttributeError, TypeError, ValueError):
+        return False
+    return True
+
+
+def _patch_unconvertible_columns() -> None:
+    """Type a column ibis cannot convert as ``Unknown`` instead of failing the model."""
+    from ibis.backends import databricks as databricks_backend
+
+    original = databricks_backend._databricks_schema_to_ibis
+    if getattr(original, _PATCHED_FLAG, False):
+        return
+    databricks_backend._databricks_schema_to_ibis = _tolerant_schema_reader(original)
+
+
+def _tolerant_schema_reader(original):
+    """Wrap ibis's ``_databricks_schema_to_ibis`` with a per-column fallback."""
+
+    def databricks_schema_to_ibis(schema):
+        try:
+            return original(schema)
+        except Exception:
+            logger.debug("Databricks schema conversion failed, retrying column by column", exc_info=True)
+        return _schema_column_by_column(original, schema)
+
+    setattr(databricks_schema_to_ibis, _PATCHED_FLAG, True)
+    return databricks_schema_to_ibis
+
+
+def _schema_column_by_column(original, schema):
+    """Convert each column on its own, typing the unconvertible ones as ``Unknown``."""
+    import ibis.expr.datatypes as dt
+    import ibis.expr.schema as sch
+
+    fields, unknown_columns = {}, []
+    for item in schema:
+        name = item["name"]
+        try:
+            fields[name] = original([item])[name]
+        except Exception:
+            fields[name] = dt.unknown
+            unknown_columns.append(f"{name} ({_type_name(item)})")
+    if unknown_columns:
+        logger.warning(
+            f"Column(s) {', '.join(unknown_columns)} have a type ibis cannot represent. "
+            f"Type checks for these columns will fail."
+        )
+    return sch.Schema(fields)
+
+
+def _type_name(item) -> str:
+    try:
+        return str(item["type"]["name"])
+    except Exception:
+        return "unknown type"

datacontract/export/sql_type_converter.py

@@ -442,6 +442,10 @@ def convert_to_databricks(field: Union[SchemaProperty, FieldLike]) -> None | str
     if base_type is None:
         return None
 
+    if base_type in ["varchar", "char"] and _get_params(field):
+        # Databricks has VARCHAR(n) / CHAR(n); collapsing them to STRING would
+        # drop the declared length.
+        return _attach_params_if_present(base_type.upper(), field)
     if base_type in ["string", "varchar", "text"]:
         return "STRING"
     if base_type in ["timestamp", "timestamp_tz"]:
@@ -488,8 +492,11 @@ def convert_to_databricks(field: Union[SchemaProperty, FieldLike]) -> None | str
         return "ARRAY<STRING>"
     if base_type in ["variant"]:
         return "VARIANT"
-    if _get_params(field):
-        return _get_type(field)
+    # A parameterized type with no mapping (map<string,int>, geography(4326)) is
+    # already spelled the way Databricks declares it, so pass it through.
+    field_type = _get_type(field)
+    if field_type and ("(" in field_type or "<" in field_type):
+        return field_type
     return _warn_cannot_map_type(field, "databricks")
 
 

datacontract/imports/unity_importer.py

@@ -175,7 +175,6 @@ def _to_property(column: ColumnInfo) -> SchemaProperty:
         required=required if required else None,
         properties=nested_properties,
         items=items,
-        custom_properties={"databricksType": sql_type} if sql_type else None,
     )
 
 

docs/docs/reference/databricks.md

@@ -39,7 +39,7 @@ The authentication method is selected from the variables you set, in this order:
 
 ### Importing
 
-`datacontract import unity` keeps the full Unity Catalog type text as `physicalType` (e.g. `decimal(10,2)`, `struct<x:int>`, `map<string,int>`) — also duplicated in the `databricksType` custom property — and maps to `logicalType`:
+`datacontract import unity` keeps the full Unity Catalog type text as `physicalType` (e.g. `decimal(10,2)`, `struct<x:int>`, `map<string,int>`) and maps to `logicalType`:
 
 | Databricks type | `logicalType` |
 |---|---|

docs/docs/release-notes.md

@@ -26,6 +26,13 @@ marked as such in the entry.
 
 ## Unreleased {#unreleased}
 
+### Changed
+- `datacontract import unity` no longer writes the `databricksType` custom property, which duplicated `physicalType`
+- `datacontract export sql --server databricks` keeps the declared length of `varchar(n)` and `char(n)` instead of exporting `STRING`
+
+### Fixed
+- `datacontract test` for Databricks no longer fails all checks of a model with a `GEOGRAPHY` or `GEOMETRY` column ([#1483](https://github.com/datacontract/datacontract-cli/issues/1483))
+
 ## 1.1.0 — 2026-08-04 {#v1-1-0}
 
 This release drops the pyspark compile-time dependency. The server types `dataframe` and `databricks` still work with a provided Spark session.

tests/fixtures/databricks-unity/import/datacontract.yaml

@@ -19,44 +19,23 @@ schema:
   properties:
   - name: id
     physicalType: int
-    customProperties:
-    - property: databricksType
-      value: int
     logicalType: integer
     required: true
   - name: name
     physicalType: varchar(255)
-    customProperties:
-    - property: databricksType
-      value: varchar(255)
     logicalType: string
   - name: age
     physicalType: smallint
-    customProperties:
-    - property: databricksType
-      value: smallint
     logicalType: integer
   - name: salary
     physicalType: decimal(10,2)
-    customProperties:
-    - property: databricksType
-      value: decimal(10,2)
     logicalType: number
   - name: join_date
     physicalType: date
-    customProperties:
-    - property: databricksType
-      value: date
     logicalType: date
   - name: updated_at
     physicalType: timestamp
-    customProperties:
-    - property: databricksType
-      value: timestamp
     logicalType: timestamp
   - name: is_active
     physicalType: boolean
-    customProperties:
-    - property: databricksType
-      value: boolean
     logicalType: boolean

tests/fixtures/databricks-unity/import/datacontract_complex_types.yaml

@@ -18,35 +18,23 @@ schema:
   properties:
   - name: id
     physicalType: bigint
-    customProperties:
-    - property: databricksType
-      value: bigint
     logicalType: integer
   - name: id_list
     physicalType: array<bigint>
-    customProperties:
-    - property: databricksType
-      value: array<bigint>
     logicalType: array
     items:
       name: items
       physicalType: bigint
       logicalType: integer
   - name: id_struct
     physicalType: struct<value:bigint>
-    customProperties:
-    - property: databricksType
-      value: struct<value:bigint>
     logicalType: object
     properties:
     - name: value
       physicalType: bigint
       logicalType: integer
   - name: struct_list
     physicalType: array<struct<key:string,value:bigint>>
-    customProperties:
-    - property: databricksType
-      value: array<struct<key:string,value:bigint>>
     logicalType: array
     items:
       name: items
@@ -61,6 +49,3 @@ schema:
         logicalType: integer
   - name: attributes
     physicalType: map<string,bigint>
-    customProperties:
-    - property: databricksType
-      value: map<string,bigint>

tests/test_sql_type_converter_physicaltype.py

@@ -205,12 +205,30 @@ def test_decimal_10_2_bigquery():
 
 
 def test_varchar_100_databricks():
-    """VARCHAR(100) on databricks: 'varchar' -> 'STRING'."""
+    """VARCHAR(100) on databricks: 'varchar' accepts params -> 'VARCHAR(100)'.
+
+    Databricks has VARCHAR(n) and enforces the length; collapsing it to STRING
+    would drop the length the contract declares.
+    """
     field = SchemaProperty(name="col", physicalType="VARCHAR(100)")
     result = convert_to_sql_type(field, "databricks")
+    assert result == "VARCHAR(100)"
+
+
+def test_varchar_without_length_databricks():
+    """A VARCHAR with no length has nothing to preserve -> 'STRING'."""
+    field = SchemaProperty(name="col", physicalType="VARCHAR")
+    result = convert_to_sql_type(field, "databricks")
     assert result == "STRING"
 
 
+def test_map_databricks():
+    """MAP has no databricks mapping, but the declared type is already Databricks SQL."""
+    field = SchemaProperty(name="col", physicalType="map<string,bigint>")
+    result = convert_to_sql_type(field, "databricks")
+    assert result == "map<string,bigint>"
+
+
 def test_decimal_18_4_trino():
     """DECIMAL(18,4) on trino: 'decimal' -> 'decimal', accepts params -> 'decimal(18,4)'."""
     field = SchemaProperty(name="col", physicalType="DECIMAL(18,4)")