feat(objects): read an object's definition text on fifteen type-ids (#789)

database-compose.yml

@@ -648,9 +648,21 @@ services:
     #
     # THERE IS NO ICEBERG CATALOG HERE, and that is a deliberate limit rather than an
     # oversight. A materialized view is the one object kind Trino cannot create without one,
-    # and on 476 only a HIVE-METASTORE-backed Iceberg catalog will create one at all: the
-    # JDBC and REST catalog types both answer `createMaterializedView is not supported`. So
-    # an Iceberg catalog here would mean a metastore service, a warehouse volume and a
+    # and on 476 only a HIVE-METASTORE-backed Iceberg catalog will create one at all.
+    #
+    # THE CHEAP ROUTE WAS PROBED FOR #789 AND REFUSED, so the next person does not have to
+    # spend the same afternoon finding out. The hope was that Trino's Iceberg connector would
+    # take `iceberg.catalog.type=jdbc` against the `postgres` service above, giving a
+    # materialized-view fixture for the price of one properties file and a warehouse volume
+    # with NO new image in this file. Measured on 2026-09-13, the JDBC catalog was brought
+    # all the way up and WORKS - `CREATE SCHEMA`, `CREATE TABLE` and a two-row `INSERT` all
+    # succeeded - and `CREATE MATERIALIZED VIEW` still answered
+    # `createMaterializedView is not supported for Iceberg JDBC catalogs`. Two traps found on
+    # the way, both recorded in docs/providers/trino.md: Trino 476 never creates the JDBC
+    # catalog's own `iceberg_tables` table, and a `file://` warehouse needs
+    # `fs.hadoop.enabled=true` rather than the native local filesystem.
+    #
+    # So an Iceberg catalog here would mean a metastore service, a warehouse volume and a
     # second image in this file for one object kind. The object surface therefore answers
     # `materialized_view: 0` everywhere on this cluster, which is the TRUE answer for it, and
     # docs/providers/trino.md carries the catalog properties and the exact statements that

docker/clickhouse-init/01-object-fixture.sql

@@ -144,3 +144,32 @@ ENGINE = Dictionary(demo.dict_customers);
 -- A user-defined function. CREATE FUNCTION takes no database qualifier, because a
 -- ClickHouse UDF is SERVER-GLOBAL: system.functions has no database column at all.
 CREATE FUNCTION order_total_with_tax AS (total) -> total * 1.2;
+
+-- Two ADVERSARIAL NAMES for the source read's escaper (#789).
+--
+-- Measured on 26.7.1.1315 (#789 probe 11): a backslash inside a QUOTED IDENTIFIER is
+-- processed as an ESCAPE on this engine, in the double-quote form and the backtick form
+-- alike, so a name ending in one swallows the closing quote and the parser runs on into
+-- whatever follows. The source read therefore takes NO identifier position at all: every
+-- caller-supplied name reaches its statements as a STRING LITERAL, where the same hazard
+-- exists and the provider's `literal()` answers it by escaping the backslash as well as
+-- doubling the quote.
+--
+-- These two tables are what makes a test of that non-vacuous rather than a claim in a
+-- docblock. `bs_one\` stores exactly ONE trailing backslash (hex(name) = 62735F6F6E655C,
+-- length 7), and reading its definition back with the backslash left unescaped fails with
+-- code 62, `Single quoted string is not closed`, at the position of the clause that
+-- followed. Renaming or dropping either one makes that measurement unrepeatable.
+CREATE TABLE demo.`bs_one\\`
+(
+  x UInt8
+)
+ENGINE = MergeTree
+ORDER BY x;
+
+CREATE TABLE demo.`dq"two`
+(
+  x UInt8
+)
+ENGINE = MergeTree
+ORDER BY x;

docker/couchbase-init/01-object-fixture.sh

@@ -55,6 +55,20 @@
 #  7. `_default`.`airline` carries the SAME collection name as `inventory`.`airline`, with
 #     an index of the same name over a DIFFERENT key, so a scope-blind filter on a
 #     collection's indexes reports the wrong keys rather than merely the wrong count.
+#  8. The two `discount` functions are also what the SOURCE READ is measured on
+#     (#789 Phase 2). Measured on Server 8.0.2 Community, `system:functions`
+#     answers `inventory`.`discount` as
+#     {"#language":"inline","expression":"(`price` - ((`price` * `pct`) / 100))",
+#      "parameters":["price","pct"],"text":"price - (price * pct / 100)"}.
+#     `text` is the body AS AUTHORED and `expression` is the engine's
+#     normalisation of it, which is the measurement behind reading `text` and
+#     captioning the part `origin: "stored"`. The two bodies differ from each
+#     other, so a read that matched on the name alone answers the wrong one
+#     rather than the same one by luck. There is no EXTERNAL JavaScript function
+#     here and there cannot be: Community Edition refuses to create one
+#     ("Functions of type javascript are only supported in Enterprise Edition",
+#     measured verbatim), which is why the provider's no-body refusal branch is
+#     driven by its suite and not by this fixture.
 #
 # The `_system` scope and its collections (`_mobile`, `_query`) are the server's
 # own and are created by it, not here. They are what the `_system` exclusion is
@@ -121,7 +135,11 @@ n1ql "CREATE INDEX \`ix_name\` IF NOT EXISTS ON \`$BUCKET\`.\`_default\`.\`airli
 #    Enterprise Edition", measured, which is why the provider never classifies on
 #    definition.`#language`.
 n1ql "CREATE OR REPLACE FUNCTION \`$BUCKET\`.\`inventory\`.\`discount\`(price, pct) { price - (price * pct / 100) }"
-n1ql "CREATE OR REPLACE FUNCTION \`$BUCKET\`.\`_default\`.\`discount\`(x) { x }"
+#    The body is `x / 2` and not a single token on purpose: the object-surface
+#    conformance helper bounds the LONGEST definition it read and refuses a
+#    definition under two characters, because a one-character text cannot be told
+#    bounded from unbounded (#789).
+n1ql "CREATE OR REPLACE FUNCTION \`$BUCKET\`.\`_default\`.\`discount\`(x) { x / 2 }"
 
 # 4. A GLOBAL function, which must never reach the tree.
 n1ql "CREATE OR REPLACE FUNCTION \`celsius\`(f) { (f - 32) / 1.8 }"

docker/duckdb-init/01-object-fixture.sql

@@ -0,0 +1,69 @@
+-- The DuckDB object-surface and object-source fixture (#789).
+--
+-- Two catalogs, four schemas, and at least one object of every declared kind, so every
+-- assertion `tests/integration/db/duckdb-provider.test.ts` makes about counts, listings,
+-- detail and definition text is made about objects THIS file created.
+--
+-- Applied two ways, by one reader: the integration suite replays it into `:memory:`, and
+-- `bun docker/duckdb-init/build-fixture.ts` replays it into a database FILE a person can
+-- point Studio at. Before #789 this DDL lived only inside the suite, which is the shape
+-- standing ruling 5i forbids: a measurement nobody outside the test run can re-run.
+--
+-- THE SECOND CATALOG'S TARGET IS A PLACEHOLDER, and it has to be. `ATTACH ':memory:' AS
+-- warehouse` gives the suite a second real catalog inside one process; a database FILE
+-- needs a sibling file there instead, or every `warehouse` object vanishes the moment the
+-- builder exits and a person opening the file finds one catalog where the tests saw two.
+-- `readFixtureStatements()` substitutes {{warehouse}} and throws by name when nothing
+-- substituted it.
+--
+-- DuckDB has NO trigger and NO stored procedure (both are parser errors on v1.5.5), so
+-- neither appears below and neither is declared.
+
+CREATE SCHEMA analytics;
+
+CREATE TABLE main.customers (id INTEGER PRIMARY KEY, name VARCHAR NOT NULL, note VARCHAR DEFAULT 'none');
+CREATE TABLE main.orders (id INTEGER PRIMARY KEY, customer_id INTEGER REFERENCES main.customers(id), total DECIMAL(12,2));
+CREATE INDEX ix_orders_customer ON main.orders(customer_id);
+
+CREATE TABLE analytics.events (id BIGINT, payload VARCHAR);
+
+-- Two SAME-NAMED tables in two schemas, with different columns and an index each. This is
+-- what a detail read's schema filter is for: without it `main.customers` and
+-- `analytics.customers` merge, and the merged answer is a table with columns it does not
+-- have rather than an error anybody would notice.
+CREATE TABLE analytics.customers (event_id BIGINT);
+CREATE TABLE analytics.orders (id INTEGER);
+CREATE INDEX ix_orders_customer ON analytics.orders(id);
+
+CREATE VIEW main.customer_names AS SELECT name FROM main.customers;
+CREATE VIEW analytics.event_days AS SELECT id FROM analytics.events;
+
+-- One macro of each form. A scalar macro is `function_type` 'macro' and a table macro is
+-- 'table_macro'; they are the whole vocabulary `FUNCTION_TYPE_RULES` accounts for, and
+-- both are here so the two arms cannot rot into one.
+CREATE MACRO main.add_one(x) AS x + 1;
+CREATE MACRO analytics.recent_events(n) AS TABLE SELECT * FROM analytics.events LIMIT n;
+
+CREATE SEQUENCE main.customer_seq START 1;
+CREATE SEQUENCE analytics.event_seq START 100;
+
+-- One name, three kinds, in one schema, and it is the reason `readObjectSource` takes the
+-- KIND as well as the path. Measured on DuckDB v1.5.5: `CREATE SEQUENCE overlap` and
+-- `CREATE MACRO overlap(x)` both succeed while the table `overlap` exists, and only
+-- `CREATE VIEW overlap` is refused (`Catalog Error: Table with name "overlap" already
+-- exists!`). A source read keyed on the name alone answers one of the three at random.
+CREATE TABLE main.overlap (id INTEGER);
+CREATE SEQUENCE main.overlap;
+CREATE MACRO main.overlap(x) AS x;
+
+-- A second real catalog, with a schema set of its own and no macro or sequence, so a
+-- declared-and-empty folder has somewhere to be measured.
+ATTACH '{{warehouse}}' AS warehouse;
+CREATE SCHEMA warehouse.stock;
+CREATE TABLE warehouse.main.ledger (id INTEGER);
+-- Same SCHEMA name, same TABLE name, different CATALOG. This is the two-level engine's
+-- characteristic case and the only thing that can show the catalog filter working: a
+-- detail read that dropped `database_name = $1` would merge `memory.main.customers` with
+-- this one and report a table with columns from both.
+CREATE TABLE warehouse.main.customers (sku VARCHAR);
+CREATE TABLE warehouse.stock.items (sku VARCHAR);

docker/duckdb-init/build-fixture.ts

@@ -0,0 +1,167 @@
+/**
+ * The DuckDB object-surface and object-source fixture, read and applied (#789).
+ *
+ * Two callers and one reader. `tests/integration/db/duckdb-provider.test.ts` calls
+ * `readFixtureStatements(":memory:")` and replays the result into an in-memory database, so
+ * the objects its object-surface and Source assertions reason about are created BY
+ * `01-object-fixture.sql`; running this file as a script replays the same statements into a
+ * database FILE a person can open in Studio. Before #789 the DDL lived only inside the
+ * suite, which is the shape standing ruling 5i forbids: a live measurement nobody outside
+ * the test run can re-run.
+ *
+ * Run it:
+ *
+ *   bun docker/duckdb-init/build-fixture.ts                    # ./.duckdb-fixture/object-fixture.duckdb
+ *   bun docker/duckdb-init/build-fixture.ts /tmp/demo.duckdb   # anywhere else
+ *
+ * It is re-runnable: an existing file at the target path is removed first, together with
+ * the `.wal` sidecar a crashed session can leave beside it and the warehouse sibling below,
+ * so a second run produces a database identical to the first rather than one holding both
+ * runs' objects.
+ *
+ * THE SECOND CATALOG IS WHY THIS FILE SUBSTITUTES A PLACEHOLDER RATHER THAN JUST SPLITTING.
+ * The fixture's `ATTACH '{{warehouse}}' AS warehouse` is a second REAL catalog, and the two
+ * callers need two different targets: the suite wants `:memory:`, which lives and dies with
+ * the process, and a file build wants a sibling file, or every `warehouse` object vanishes
+ * when the builder exits and a person opening the result finds one catalog where the tests
+ * saw two.
+ *
+ * ONE LIMIT, STATED HERE RATHER THAN DISCOVERED: DuckDB does NOT persist an attachment
+ * inside a database file. So opening the built file shows the `memory` half only, and the
+ * second catalog is reached with `ATTACH '<target>.warehouse.duckdb' AS warehouse` in a
+ * query tab. The sibling is still built, because the alternative - dropping the second
+ * catalog from the file build - would give a person a fixture that cannot show the two
+ * container levels this engine's whole object model is about.
+ */
+import { DuckDBInstance } from "@duckdb/node-api";
+import * as fs from "fs";
+import * as path from "path";
+
+/** Where the fixture lives, so a caller names a file rather than a path. */
+export const FIXTURE_DIRECTORY = import.meta.dir;
+
+/** The fixture the DuckDB provider's own suite replays. */
+export const DUCKDB_FIXTURE_FILE = "01-object-fixture.sql";
+
+/** The default target, relative to the repository root. */
+export const DEFAULT_FIXTURE_PATH = ".duckdb-fixture/object-fixture.duckdb";
+
+/**
+ * The token `ATTACH` names its target with, and the one thing in this fixture that is not
+ * literal SQL.
+ *
+ * Spelled once and exported, so the substitution and the guard that it happened cannot
+ * disagree about what they are looking for.
+ */
+export const WAREHOUSE_PLACEHOLDER = "{{warehouse}}";
+
+/** The second catalog's target for a file build, derived from the target's own path. */
+export function warehouseSibling(file: string): string {
+  return `${file}.warehouse.duckdb`;
+}
+
+/**
+ * Every statement of the fixture, comments removed, in the file's own order, with the
+ * second catalog's target substituted in.
+ *
+ * A line whose first non-blank characters are `--` is dropped whole; there is no `--`
+ * inside a string literal in this fixture, and a splitter that pretended otherwise would be
+ * claiming a lexer it does not have. A statement ends at a line ending in `;`, which is
+ * enough here for a reason DuckDB gives rather than luck: this engine has no trigger and no
+ * stored procedure, so no statement in the fixture carries a body holding its own
+ * semicolon. That is the trap `docker/sqlite-init/build-fixture.ts` needs a trigger rule
+ * for, and it cannot arise on this engine.
+ *
+ * THREE THROWS, each over a zero that would otherwise pass silently:
+ *
+ * - no statement at all, which builds an EMPTY database and makes every count assertion
+ *   downstream read zero against zero. The same throw covers a path resolving to the wrong
+ *   file;
+ * - no substitution, which would send DuckDB the literal string `{{warehouse}}` as a file
+ *   PATH. Measured on v1.5.5: that ATTACH succeeds and creates a file called
+ *   `{{warehouse}}` in the working directory, so the failure is silent litter rather than
+ *   an error, and the suite would still see two catalogs;
+ * - an empty warehouse target, because `ATTACH '' AS warehouse` is the same shape one step
+ *   further along.
+ */
+export function readFixtureStatements(warehouse: string, file: string = DUCKDB_FIXTURE_FILE): string[] {
+  if (warehouse.trim() === "") {
+    throw new Error("the warehouse target is blank, so the fixture's second catalog would attach nothing");
+  }
+  const resolved = path.isAbsolute(file) ? file : path.join(FIXTURE_DIRECTORY, file);
+  const text = fs.readFileSync(resolved, "utf8");
+  const statements: string[] = [];
+  let buffer = "";
+  for (const line of text.split("\n")) {
+    if (line.trim().startsWith("--")) continue;
+    const body = line.trimEnd();
+    if (!body.endsWith(";")) {
+      buffer += `${line}\n`;
+      continue;
+    }
+    const statement = `${buffer}${body.slice(0, -1)}`.trim();
+    if (statement !== "") statements.push(statement);
+    buffer = "";
+  }
+  if (statements.length === 0) {
+    throw new Error(`${resolved} yielded no statement, so applying it would build an empty database`);
+  }
+  const substituted = statements.map((statement) => statement.split(WAREHOUSE_PLACEHOLDER).join(warehouse));
+  if (substituted.every((statement, index) => statement === statements[index])) {
+    throw new Error(
+      `${resolved} carries no ${WAREHOUSE_PLACEHOLDER} to substitute, so the second catalog would attach a file ` +
+        "named after the placeholder itself",
+    );
+  }
+  return substituted;
+}
+
+/**
+ * Delete a DuckDB database, the `.wal` sidecar, and the warehouse sibling beside it.
+ *
+ * ABSENCE IS THE ONLY FAILURE THIS SWALLOWS, and the narrowing is the point: the file not
+ * being there IS the state this function is asked for, while any other errno means the file
+ * is still on disk. A bare `catch {}` reported success there, and `buildObjectFixture` then
+ * opened the STALE database and replayed the fixture DDL on top of it, so the first failure
+ * a person saw was `CREATE TABLE main.customers` raising a catalog error that named nothing
+ * about an undeleted file. Raising here is also the house rule: no silent recovery and no
+ * symptom-masking guard.
+ */
+export function removeDatabaseFile(file: string): void {
+  for (const target of [file, `${file}.wal`, warehouseSibling(file), `${warehouseSibling(file)}.wal`]) {
+    try {
+      fs.unlinkSync(target);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+    }
+  }
+}
+
+/** Build the fixture at `file`, replacing whatever is there, with its warehouse sibling. */
+export async function buildObjectFixture(file: string): Promise<void> {
+  removeDatabaseFile(file);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  const instance = await DuckDBInstance.create(file);
+  const connection = await instance.connect();
+  try {
+    for (const statement of readFixtureStatements(warehouseSibling(file))) await connection.run(statement);
+    // Both catalogs, so neither file is left holding an uncheckpointed write-ahead log a
+    // reader would have to replay. The first is spelled bare rather than by name: a DuckDB
+    // file's catalog is named after its own stem, so the connected catalog here is
+    // `object-fixture` and never the `memory` the in-process suite sees.
+    await connection.run("CHECKPOINT");
+    await connection.run("CHECKPOINT warehouse");
+  } finally {
+    connection.closeSync();
+    instance.closeSync();
+  }
+}
+
+if (import.meta.main) {
+  const target = path.resolve(process.argv[2] ?? DEFAULT_FIXTURE_PATH);
+  await buildObjectFixture(target);
+  process.stdout.write(
+    `DuckDB object fixture written to ${target}\n` +
+      `Second catalog: ${warehouseSibling(target)} (ATTACH '${warehouseSibling(target)}' AS warehouse)\n`,
+  );
+}