From eef22c38a6e972e2879e43e4fa92843ba85e0ef4 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 15 Aug 2026 01:30:43 -0700 Subject: [PATCH 01/21] fix(workbench): share scan history across linked worktrees --- README.md | 25 +- sdk/typescript/README.md | 40 +- .../_bundled_plugin/scripts/workbench_cli.py | 1 + .../scripts/workbench_feedback.py | 32 +- .../scripts/workbench_native_indexes.py | 137 +++- .../scripts/workbench_scan_history.py | 312 +++++++-- .../scripts/workbench_schema.py | 124 +++- .../scripts/workbench_target_state.py | 241 ++++++- sdk/typescript/src/api.ts | 4 +- sdk/typescript/src/cli.ts | 66 +- sdk/typescript/tests-ts/cli-workbench.test.ts | 198 +++++- .../repository-feedback-aliases.test.ts | 200 ++++++ .../repository-findings-worktrees.test.ts | 177 +++++ .../tests-ts/repository-findings.test.ts | 227 +++++++ .../workbench-repository-identity.test.ts | 639 ++++++++++++++++++ .../workbench-target-migration.test.ts | 389 +++++++++++ 16 files changed, 2681 insertions(+), 131 deletions(-) create mode 100644 sdk/typescript/tests-ts/repository-feedback-aliases.test.ts create mode 100644 sdk/typescript/tests-ts/repository-findings-worktrees.test.ts create mode 100644 sdk/typescript/tests-ts/workbench-repository-identity.test.ts create mode 100644 sdk/typescript/tests-ts/workbench-target-migration.test.ts diff --git a/README.md b/README.md index 353830ea0..406b95950 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ container credentials, and the default AWS credential chain. Local sign-in honors Codex's configured credential backend, including a system keyring required by a managed device. Codex Security keeps login and scan -credentials in the same private, persistent state directory. +sign-in scoped to a dedicated Codex home in the same private, persistent state +directory. If both a ChatGPT sign-in and an API key are available, interactive scans ask which credential to use. CI and other noninteractive scans keep the existing @@ -67,12 +68,26 @@ keys: unset OPENAI_API_KEY CODEX_API_KEY ``` -Scan history is stored in the Codex Security workbench state directory. If that -directory cannot be written, set `CODEX_SECURITY_STATE_DIR` to a writable -directory outside the repository. +Scan history and saved findings are stored in one Codex Security state database. +Linked Git worktrees are discovered automatically and grouped when they use the +same state directory. Leave `CODEX_SECURITY_STATE_DIR` unset, or select one +stable, writable directory outside the repository. Changing or unsetting it +selects separate history and an isolated Codex credential home and sign-in +scope; restore the previous value to reopen its existing scans and sign-in. + +```bash +# Run from another linked Git worktree: +npx @openai/codex-security scans list +npx @openai/codex-security findings list + +# Reopen an existing, separately selected state directory: +export CODEX_SECURITY_STATE_DIR=/path/to/existing/codex-security-state +npx @openai/codex-security scans list +``` `findings list [repository]` shows open findings across a repository's scans -and identifies findings not confirmed in its latest scan. +and identifies findings not confirmed in its latest completed scan across +linked worktrees. `scans compare BEFORE_SCAN_ID AFTER_SCAN_ID` automatically matches findings by root cause, reuses saved matches, and identifies new, persisting, reopened, diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 5c2cf0698..1f30747b7 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -426,7 +426,7 @@ The CLI and SDK recognize the following user-configurable environment: | `OPENAI_API_KEY`, `CODEX_API_KEY` | Scan authentication; `OPENAI_API_KEY` wins when both are present. | | `CODEX_SECURITY_LOG_LEVEL` | CLI-only; set to `debug` for verbose diagnostics. | | `LOG_LEVEL` | CLI-only fallback when `CODEX_SECURITY_LOG_LEVEL` is unset. | -| `CODEX_SECURITY_STATE_DIR` | Override the private scan-history, workbench, and default artifact directory. | +| `CODEX_SECURITY_STATE_DIR` | Select the history database, artifact directory, and dedicated Codex credential home. | | `CODEX_HOME` | Set the ambient Codex home for file-backed sign-in and default state; defaults to `~/.codex`. | | `CODEX_CLI_PATH` | Use another Codex executable for authentication, plugin setup, scans, and nested workers. | | `PYTHON` | Select a Python interpreter when `--python` or SDK `pythonPath` is not set. | @@ -538,11 +538,15 @@ same command to resume. ### Scan history and reruns -`npx @openai/codex-security scans list` lists scans for the current repository. Pass a -repository path to inspect another checkout, `--scan-root DIR` to list scans -whose artifacts are under a particular root. `scans show SCAN_ID` includes the -scan configuration, results, coverage, and artifact locations. Add -`--show-linked-findings` to include finding links from previous scans. +`npx @openai/codex-security scans list` lists scans for the current repository. +Linked Git worktrees are discovered automatically and grouped when they share +the selected state database. Findings indicate whether they were confirmed in +the repository's latest completed scan across those linked worktrees. Pass a +repository path to inspect another checkout. `--scan-root DIR` only filters +scans already recorded in that database by their artifact directory; it never +imports scan results from another state directory. `scans show SCAN_ID` +includes the scan configuration, results, coverage, and artifact locations. +Add `--show-linked-findings` to include finding links from previous scans. `scans logs SCAN_ID` shows complete session events from the scan and its workers, which can include source code and credentials. @@ -553,9 +557,27 @@ least eight characters. Scan history uses `$CODEX_SECURITY_STATE_DIR/workbench.sqlite3` when `CODEX_SECURITY_STATE_DIR` is set. Otherwise, it uses `$CODEX_HOME/state/plugins/codex-security/workbench.sqlite3`; `CODEX_HOME` -defaults to `~/.codex`. Scan credentials are never stored in the scan -configuration. Recorded failure summaries and bulk-scan receipts omit messages -that contain recognizable credentials. +defaults to `~/.codex`. Saved findings use the same selected database. +Changing or unsetting `CODEX_SECURITY_STATE_DIR` selects separate scan history +and an isolated Codex credential home and sign-in scope; scans from the +previous state remain hidden until you select that state again. Keep the +setting stable across linked worktrees to share scans and findings +automatically. + +```bash +# Inspect shared history from another linked Git worktree: +npx @openai/codex-security scans list +npx @openai/codex-security findings list + +# Reopen an existing state directory and its sign-in: +export CODEX_SECURITY_STATE_DIR=/path/to/existing/codex-security-state +npx @openai/codex-security scans list +npx @openai/codex-security findings list +``` + +Scan credentials are never stored in the scan configuration. Recorded failure +summaries and bulk-scan receipts omit messages that contain recognizable +credentials. The scan sandbox permits writes to the selected state directory so SQLite can maintain its database and journal files. If the host itself cannot write to the diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py b/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py index 56f8997c7..15cc36115 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py @@ -131,6 +131,7 @@ def parse_args(description: str) -> argparse.Namespace: list_scans.add_argument("--status", choices=("running", "complete", "failed", "canceled")) list_scans.add_argument("--mode", choices=MODES) list_scans.add_argument("--repository") + list_scans.add_argument("--worktrees-only", action="store_true", help=argparse.SUPPRESS) list_scans.add_argument("--scan-root") list_scans.add_argument("--offset", type=non_negative_int, default=0) list_scans.add_argument("--limit", type=positive_int) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py b/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py index 51b08a161..ac3e1651c 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py @@ -16,12 +16,24 @@ FINDING_SUMMARY_BYTES, FINDING_TITLE_BYTES, ) +from workbench_native_indexes import _indexed_findings, repository_target_ids from workbench_validation import bounded_output_text def get_scan_feedback(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict[str, Any]: + target_ids = sorted(repository_target_ids(connection, scan["target_id"])) + if not target_ids: + return {"scanId": scan["id"], "targetId": scan["target_id"], "falsePositives": []} + + indexed_findings = { + finding_id: finding + for finding in _indexed_findings(connection) + if finding["target_id"] in target_ids + for finding_id in finding["matched_finding_ids"] + } + target_placeholders = ", ".join("?" for _ in target_ids) rows = connection.execute( - """ + f""" WITH ranked_decisions AS ( SELECT findings.id AS finding_id, findings.fingerprint, findings.rule_id, findings.identity_anchor, findings.identity_instance, occurrences.title, @@ -49,7 +61,7 @@ def get_scan_feedback(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict candidate.sort_order LIMIT 1 ) - WHERE source_scans.target_id = ? + WHERE source_scans.target_id IN ({target_placeholders}) AND source_scans.id != ? AND source_scans.status = 'complete' ) @@ -61,12 +73,22 @@ def get_scan_feedback(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict AND note IS NOT NULL AND trim(note) != '' ORDER BY updated_at DESC, source_completed_at DESC, source_scan_id DESC, finding_id DESC - LIMIT 50 """, - (scan["target_id"], scan["id"]), + (*target_ids, scan["id"]), ) false_positives = [] + reviewed_components: set[str] = set() for row in rows: + finding = indexed_findings.get(row["finding_id"]) + if ( + finding is None + or finding["status"] != "closed" + or finding["close_reason"] != "false_positive" + or finding["occurrence_id"] in reviewed_components + ): + continue + reviewed_components.add(finding["occurrence_id"]) + identity = {"anchor": row["identity_anchor"]} if row["identity_instance"] is not None: identity["instance"] = row["identity_instance"] @@ -91,6 +113,8 @@ def get_scan_feedback(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict "updatedAt": row["updated_at"], } ) + if len(false_positives) == 50: + break return {"scanId": scan["id"], "targetId": scan["target_id"], "falsePositives": false_positives} diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py index 8ce2492bc..f341af1a3 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py @@ -13,19 +13,65 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import workbench_scan_history as scan_history from workbench_constants import FINDING_SUMMARY_BYTES, FINDING_TITLE_BYTES, FINDINGS_PAGE_MAX +from workbench_target_state import _require_current_target_owner from workbench_validation import bounded_output_text +def repository_target_ids(connection: sqlite3.Connection, target_id: str) -> set[str]: + if not _has_repository_identities(connection): + return {target_id} + + requested_target = connection.execute( + "SELECT current_path, repository_identity FROM security_targets WHERE id = ?", + (target_id,), + ).fetchone() + if requested_target is None: + return {target_id} + try: + _require_current_target_owner( + connection, + target_id, + requested_target["current_path"], + requested_target["repository_identity"], + ) + except SystemExit: + return set() + + rows = connection.execute( + """ + SELECT id + FROM security_targets + WHERE id = ? + OR ( + repository_identity IS NOT NULL + AND repository_identity = ? + ) + """, + (target_id, requested_target["repository_identity"]), + ) + return {row["id"] for row in rows} or {target_id} + + +def _has_repository_identities(connection: sqlite3.Connection) -> bool: + return any( + column["name"] == "repository_identity" + for column in connection.execute("PRAGMA table_info(security_targets)") + ) + + def list_global_findings( connection: sqlite3.Connection, args: argparse.Namespace, ) -> dict[str, Any]: limit = min(args.limit, FINDINGS_PAGE_MAX) query = args.query.strip().casefold() if args.query else "" + target_ids = ( + None if args.target_id is None else repository_target_ids(connection, args.target_id) + ) findings = ( row for row in _indexed_findings(connection) - if (args.target_id is None or row["target_id"] == args.target_id) + if (target_ids is None or row["target_id"] in target_ids) and (args.severity is None or row["severity"] == args.severity) and (args.status is None or row["status"] == args.status) and ( @@ -75,40 +121,85 @@ def list_global_findings( def _indexed_findings(connection: sqlite3.Connection) -> Iterator[dict[str, Any]]: - parents: dict[tuple[str, str], tuple[str, str]] = {} + parents: dict[ + tuple[tuple[str, str], str], tuple[tuple[str, str], str] + ] = {} + has_repository_identities = _has_repository_identities(connection) + identity_column = "targets.repository_identity" if has_repository_identities else "NULL" + before_identity_column = ( + "before_targets.repository_identity" if has_repository_identities else "NULL" + ) + after_identity_column = ( + "after_targets.repository_identity" if has_repository_identities else "NULL" + ) + alias_condition = ( + "before_targets.repository_identity IS NOT NULL " + "AND before_targets.repository_identity = after_targets.repository_identity" + if has_repository_identities + else "0" + ) - def group(identity: tuple[str, str]) -> tuple[str, str]: + def repository_identity(target_id: str, identity: str | None) -> tuple[str, str]: + return ("target", target_id) if identity is None else ("repository", identity) + + def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: while identity in parents: identity = parents[identity] return identity for match in connection.execute( - """ - SELECT before_scans.target_id, before.finding_id AS before_finding_id, + f""" + SELECT before_scans.target_id AS before_target_id, + after_scans.target_id AS after_target_id, + {before_identity_column} AS before_repository_identity, + {after_identity_column} AS after_repository_identity, + before.finding_id AS before_finding_id, after.finding_id AS after_finding_id FROM scan_comparison_matches AS matches JOIN finding_occurrences AS before ON before.id = matches.before_occurrence_id JOIN scans AS before_scans ON before_scans.id = before.scan_id + JOIN security_targets AS before_targets ON before_targets.id = before_scans.target_id JOIN finding_occurrences AS after ON after.id = matches.after_occurrence_id JOIN scans AS after_scans ON after_scans.id = after.scan_id - WHERE before_scans.target_id = after_scans.target_id + JOIN security_targets AS after_targets ON after_targets.id = after_scans.target_id + WHERE before_scans.target_id = after_scans.target_id OR ({alias_condition}) """ ): - before = group((match["target_id"], match["before_finding_id"])) - after = group((match["target_id"], match["after_finding_id"])) + before = group( + ( + repository_identity( + match["before_target_id"], match["before_repository_identity"] + ), + match["before_finding_id"], + ) + ) + after = group( + ( + repository_identity( + match["after_target_id"], match["after_repository_identity"] + ), + match["after_finding_id"], + ) + ) if before != after: parents[after] = before - latest_scan_by_target = dict( - connection.execute( - "SELECT target_id, id FROM scans " - "WHERE status = 'complete' ORDER BY started_at, id" + latest_scan_by_repository = { + repository_identity(row["target_id"], row["repository_identity"]): row["id"] + for row in connection.execute( + f""" + SELECT scans.target_id, scans.id, {identity_column} AS repository_identity + FROM scans + JOIN security_targets AS targets ON targets.id = scans.target_id + WHERE scans.status = 'complete' + ORDER BY scans.started_at, scans.id + """ ) - ) + } - grouped: dict[tuple[str, str], list[sqlite3.Row]] = {} + grouped: dict[tuple[tuple[str, str], str], list[sqlite3.Row]] = {} for row in connection.execute( - """ + f""" SELECT occurrences.id AS occurrence_id, occurrences.finding_id, @@ -118,6 +209,7 @@ def group(identity: tuple[str, str]) -> tuple[str, str]: scans.started_at AS scan_started_at, scans.target_id, targets.current_path AS target_path, + {identity_column} AS repository_identity, scans.scope, MAX(scans.updated_at, COALESCE(triage.updated_at, '')) AS updated_at, triage.status AS decision_status, @@ -140,7 +232,15 @@ def group(identity: tuple[str, str]) -> tuple[str, str]: LEFT JOIN finding_triage AS triage ON triage.occurrence_id = occurrences.id """, ): - grouped.setdefault(group((row["target_id"], row["finding_id"])), []).append(row) + grouped.setdefault( + group( + ( + repository_identity(row["target_id"], row["repository_identity"]), + row["finding_id"], + ) + ), + [], + ).append(row) findings = [] for occurrences in grouped.values(): @@ -161,7 +261,10 @@ def group(identity: tuple[str, str]) -> tuple[str, str]: findings.append( { **dict(latest), - "confirmed_in_latest_scan": latest_scan_by_target.get(latest["target_id"]) + "close_reason": decision["close_reason"] if decision is not None else None, + "confirmed_in_latest_scan": latest_scan_by_repository.get( + repository_identity(latest["target_id"], latest["repository_identity"]) + ) == latest["scan_id"], "known_since": scans[0][0], "known_scan_ids": [scan_id for _, scan_id in scans], diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py index 323fdbe78..ece0e2cfb 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py @@ -12,40 +12,68 @@ # Some plugin hosts launch Python with safe-path isolation enabled. sys.path.insert(0, str(Path(__file__).resolve().parent)) +from filesystem_identity import serialize_filesystem_identity from report_projection import SEVERITY_ORDER from workbench_constants import FINDINGS_PAGE_MAX from workbench_scan_usage import stored_scan_cost_fields from workbench_target import git_output +from workbench_target_state import ( + _require_current_target_owner, + repository_identity, + repository_relative_path, +) def _same_repository( before: sqlite3.Row, after: sqlite3.Row, *, - after_identity: tuple[str | None, tuple[str, str] | None] | None = None, + after_identity: tuple[str | None, tuple[str, str] | None, str | None] | None = None, + worktrees_only: bool = False, ) -> bool: - if before["target_id"] == after["target_id"]: + before_target_id = before["target_id"] + if before_target_id and before_target_id == after["target_id"]: return True before_target = Path(before["target_path"]) after_target = Path(after["target_path"]) - before_git_dir = git_output( - before_target, "rev-parse", "--path-format=absolute", "--git-common-dir" + if before_target.resolve() == after_target.resolve(): + return True + before_stored_identity = ( + before["repository_identity"] if "repository_identity" in before.keys() else None + ) + after_stored_identity = ( + after["repository_identity"] if "repository_identity" in after.keys() else None ) - after_git_dir = ( - git_output(after_target, "rev-parse", "--path-format=absolute", "--git-common-dir") - if after_identity is None + after_grouping_identity = ( + after_stored_identity + if after_stored_identity is not None else after_identity[0] + if after_identity is not None + else None ) - if ( - before_git_dir is not None - and after_git_dir is not None - and Path(before_git_dir).resolve() == Path(after_git_dir).resolve() - ): + if before_stored_identity and before_stored_identity == after_grouping_identity: return True + + before_live_identity = repository_identity(before_target) + after_live_identity = ( + repository_identity(after_target) if after_identity is None else after_identity[0] + ) + if before_live_identity is not None and before_live_identity == after_live_identity: + return True + if worktrees_only or before_live_identity is None or after_live_identity is None: + return False + before_origin = _repository_origin(before_target) - return before_origin is not None and before_origin == ( + after_origin = ( _repository_origin(after_target) if after_identity is None else after_identity[1] ) + if before_origin is None or before_origin != after_origin: + return False + before_relative = repository_relative_path(before_target) + after_relative = ( + repository_relative_path(after_target) if after_identity is None else after_identity[2] + ) + return before_relative is not None and before_relative == after_relative def _repository_origin(target: Path) -> tuple[str, str] | None: @@ -82,30 +110,99 @@ def list_scans( values: list[Any] = [] if args is not None and args.repository: repository = Path(args.repository).expanduser().resolve() + worktrees_only = bool(getattr(args, "worktrees_only", False)) + target_columns = { + row["name"] for row in connection.execute("PRAGMA table_info(security_targets)") + } + identity_column = "repository_identity" in target_columns + requested_identity_column = ( + ", (SELECT repository_identity FROM security_targets " + "WHERE current_path = ?) AS repository_identity" + if identity_column + else "" + ) + requested_values = ( + (str(repository), str(repository), str(repository)) + if identity_column + else (str(repository), str(repository)) + ) requested_repository = connection.execute( - """ + f""" SELECT COALESCE((SELECT id FROM security_targets WHERE current_path = ?), '') AS target_id, ? AS target_path + {requested_identity_column} """, - (str(repository), str(repository)), + requested_values, ).fetchone() + scan_columns = { + row["name"] for row in connection.execute("PRAGMA table_info(scans)") + } + check_ownership = {"target_device", "target_inode"} <= scan_columns + ownership_matches = not check_ownership or _repository_ownership_matches( + connection, + repository, + requested_repository["target_id"], + requested_repository["repository_identity"] if identity_column else None, + ) + live_identity = repository_identity(repository) + identity_matches = ( + not identity_column + or requested_repository["repository_identity"] is None + or requested_repository["repository_identity"] == live_identity + ) + ownership_matches = ownership_matches and identity_matches requested_identity = ( - git_output(repository, "rev-parse", "--path-format=absolute", "--git-common-dir"), - _repository_origin(repository), + ( + requested_repository["repository_identity"] + if ( + identity_column + and ownership_matches + and requested_repository["repository_identity"] is not None + ) + else live_identity + ), + None if worktrees_only else _repository_origin(repository), + None if worktrees_only else repository_relative_path(repository), ) - related_target_ids = [ - target["target_id"] - for target in connection.execute( - "SELECT id AS target_id, current_path AS target_path FROM security_targets" - ) - if _same_repository(target, requested_repository, after_identity=requested_identity) - ] - repository_clauses = ["scans.target_path = ?"] - values.append(str(repository)) - if related_target_ids: - placeholders = ", ".join("?" for _ in related_target_ids) + related_targets = ( + [ + target + for target in connection.execute( + "SELECT id AS target_id, current_path AS target_path, repository_identity " + "FROM security_targets" + if identity_column + else "SELECT id AS target_id, current_path AS target_path FROM security_targets" + ) + if _same_repository( + target, + requested_repository, + after_identity=requested_identity, + worktrees_only=worktrees_only, + ) + ] + if ownership_matches + else [] + ) + repository_clauses = ( + [_owned_scan_clause(repository, check_ownership, values)] + if identity_matches + else ["0"] + ) + if worktrees_only: + for target in related_targets: + target_path = Path(target["target_path"]) + repository_clauses.append( + _owned_scan_clause( + target_path, + check_ownership, + values, + target_id=target["target_id"], + ) + ) + elif related_targets: + placeholders = ", ".join("?" for _ in related_targets) repository_clauses.append(f"scans.target_id IN ({placeholders})") - values.extend(related_target_ids) + values.extend(target["target_id"] for target in related_targets) clauses.append(f"({' OR '.join(repository_clauses)})") if args is not None and args.scan_root: scan_root = str(Path(args.scan_root).expanduser().resolve()) @@ -219,6 +316,71 @@ def list_scans( return result +def _owned_scan_clause( + target: Path, + check_ownership: bool, + values: list[Any], + *, + target_id: str | None = None, +) -> str: + if target_id is None: + values.append(str(target)) + target_clause = "scans.target_path = ?" + else: + values.append(target_id) + target_clause = "scans.target_id = ?" + if not check_ownership: + return target_clause + try: + metadata = target.stat() + except OSError: + return target_clause + values.extend( + ( + serialize_filesystem_identity(metadata.st_dev), + serialize_filesystem_identity(metadata.st_ino), + ) + ) + return ( + f"({target_clause} AND (" + "(scans.target_device IS NULL AND scans.target_inode IS NULL) " + "OR (scans.target_device = ? AND scans.target_inode = ?)" + "))" + ) + + +def _repository_ownership_matches( + connection: sqlite3.Connection, + repository: Path, + target_id: str | None, + stored_identity: str | None = None, +) -> bool: + try: + metadata = repository.stat() + except OSError: + return False + scans = connection.execute( + """ + SELECT target_device, target_inode + FROM scans + WHERE target_path = ? OR target_id = ? + """, + (str(repository), target_id), + ) + current_device = serialize_filesystem_identity(metadata.st_dev) + current_inode = serialize_filesystem_identity(metadata.st_ino) + historical_scan = False + verified_owner = False + for scan in scans: + historical_scan = True + if scan["target_device"] is None and scan["target_inode"] is None: + continue + if scan["target_device"] != current_device or scan["target_inode"] != current_inode: + return False + verified_owner = True + return not historical_scan or verified_owner or stored_identity is None + + def list_unmatched_scan_pairs( connection: sqlite3.Connection, args: argparse.Namespace, @@ -227,19 +389,63 @@ def list_unmatched_scan_pairs( read_coverage: Callable[[sqlite3.Row], dict[str, Any]], ) -> dict[str, Any]: repository = Path(args.repository).expanduser().resolve() + identity_column = any( + row["name"] == "repository_identity" + for row in connection.execute("PRAGMA table_info(security_targets)") + ) + requested_identity_column = ( + ", (SELECT repository_identity FROM security_targets " + "WHERE current_path = ?) AS repository_identity" + if identity_column + else "" + ) + requested_values = ( + (str(repository), str(repository), str(repository)) + if identity_column + else (str(repository), str(repository)) + ) requested = connection.execute( - """ + f""" SELECT COALESCE((SELECT id FROM security_targets WHERE current_path = ?), '') AS target_id, ? AS target_path + {requested_identity_column} """, - (str(repository), str(repository)), + requested_values, ).fetchone() + if identity_column and requested["target_id"]: + _require_current_target_owner( + connection, + requested["target_id"], + str(repository), + requested["repository_identity"], + ) + scan_query = ( + "SELECT scans.*, targets.repository_identity " + "FROM scans LEFT JOIN security_targets AS targets ON targets.id = scans.target_id " + "WHERE scans.status = 'complete' ORDER BY scans.started_at, scans.id" + if identity_column + else "SELECT * FROM scans WHERE status = 'complete' ORDER BY started_at, id" + ) + live_identity = repository_identity(repository) + requested_identity = ( + requested["repository_identity"] + if identity_column and requested["repository_identity"] is not None + else live_identity + ) + requested_group = ( + requested_identity, + _repository_origin(repository), + repository_relative_path(repository), + ) selected = [ scan - for scan in connection.execute( - "SELECT * FROM scans WHERE status = 'complete' ORDER BY started_at, id" + for scan in connection.execute(scan_query) + if Path(scan["target_path"]).resolve() == repository + or _same_repository( + scan, + requested, + after_identity=requested_group, ) - if Path(scan["target_path"]).resolve() == repository or _same_repository(scan, requested) ] available = [] @@ -305,8 +511,12 @@ def compare_scans( include_matching_inputs: bool = False, require_matches: bool = False, ) -> dict[str, Any]: - before = require_scan(connection, args.before_scan_id) - after = require_scan(connection, args.after_scan_id) + before = _scan_with_repository_identity( + connection, require_scan(connection, args.before_scan_id) + ) + after = _scan_with_repository_identity( + connection, require_scan(connection, args.after_scan_id) + ) if before["id"] == after["id"]: raise SystemExit("Select two different scans to compare.") if before["status"] != "complete" or after["status"] != "complete": @@ -445,8 +655,12 @@ def save_scan_comparison( require_scan: Callable[[sqlite3.Connection, str], sqlite3.Row], read_coverage: Callable[[sqlite3.Row], dict[str, Any]], ) -> dict[str, Any]: - before = require_scan(connection, args.before_scan_id) - after = require_scan(connection, args.after_scan_id) + before = _scan_with_repository_identity( + connection, require_scan(connection, args.before_scan_id) + ) + after = _scan_with_repository_identity( + connection, require_scan(connection, args.after_scan_id) + ) if before["id"] == after["id"]: raise SystemExit("Select two different scans to compare.") if before["status"] != "complete" or after["status"] != "complete": @@ -528,6 +742,28 @@ def save_scan_comparison( return compare_scans(connection, args, require_scan=require_scan, read_coverage=read_coverage) +def _scan_with_repository_identity( + connection: sqlite3.Connection, scan: sqlite3.Row +) -> sqlite3.Row: + if "repository_identity" in scan.keys(): + return scan + if not any( + row["name"] == "repository_identity" + for row in connection.execute("PRAGMA table_info(security_targets)") + ): + return scan + enriched = connection.execute( + """ + SELECT scans.*, targets.repository_identity + FROM scans + LEFT JOIN security_targets AS targets ON targets.id = scans.target_id + WHERE scans.id = ? + """, + (scan["id"],), + ).fetchone() + return enriched if enriched is not None else scan + + def finding_matches( connection: sqlite3.Connection, occurrence_id: str, scan_id: str, started_at: str ) -> tuple[list[dict[str, Any]], str, list[str]]: diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py index 93b8f297e..50074a0b7 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py @@ -643,6 +643,17 @@ ADD COLUMN max_time_hours REAL NOT NULL DEFAULT 96; """, ), + ( + 30, + "persist repository identities", + """ + ALTER TABLE security_targets + ADD COLUMN repository_identity TEXT; + + CREATE INDEX security_targets_by_repository_identity + ON security_targets(repository_identity); + """, + ), ) @@ -704,11 +715,19 @@ def apply_migrations( "max_time_hours", "REAL NOT NULL DEFAULT 96", ) + elif version == 30: + should_backfill_targets = ( + repair_repository_identity_migration(connection) + or should_backfill_targets + ) continue if version == 6: repair_thread_scoped_workspaces_migration(connection) elif version == 16: should_backfill_targets = repair_stable_targets_migration(connection) + elif version == 30: + repair_repository_identity_migration(connection) + should_backfill_targets = True else: for statement in sql_statements(sql): connection.execute(statement) @@ -1154,38 +1173,85 @@ def repair_stable_targets_migration(connection: sqlite3.Connection) -> bool: and "target_id" in scan_columns and existing_objects == {"security_targets", "scans_by_target"} ): + needs_backfill = bool( + connection.execute( + """ + SELECT EXISTS ( + SELECT 1 FROM workspaces AS workspace + WHERE (workspace.target_id IS NULL AND workspace.target_path IS NOT NULL) + OR ( + workspace.target_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM security_targets + WHERE security_targets.id = workspace.target_id + ) + ) + ) OR EXISTS ( + SELECT 1 FROM scans AS scan + WHERE scan.target_id IS NULL + OR NOT EXISTS ( + SELECT 1 FROM security_targets + WHERE security_targets.id = scan.target_id + ) + ) + """ + ).fetchone()[0] + ) + if not needs_backfill: + return False + else: + migration_sql = next(sql for version, _, sql in MIGRATIONS if version == 16) + for statement in sql_statements(migration_sql): + if statement.startswith("ALTER TABLE workspaces"): + add_column_if_missing( + connection, + "workspaces", + "target_id", + "TEXT REFERENCES security_targets(id)", + ) + continue + if statement.startswith("ALTER TABLE scans"): + add_column_if_missing( + connection, + "scans", + "target_id", + "TEXT REFERENCES security_targets(id)", + ) + continue + statement = statement.replace("CREATE TABLE ", "CREATE TABLE IF NOT EXISTS ", 1) + statement = statement.replace("CREATE INDEX ", "CREATE INDEX IF NOT EXISTS ", 1) + connection.execute(statement) + + for table in ("workspaces", "scans"): + connection.execute( + f""" + UPDATE {table} + SET target_id = NULL + WHERE target_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM security_targets + WHERE security_targets.id = {table}.target_id + ) + """ + ) + return True + + +def repair_repository_identity_migration(connection: sqlite3.Connection) -> bool: + columns = { + row["name"] for row in connection.execute("PRAGMA table_info(security_targets)") + } + index_exists = connection.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'index' " + "AND name = 'security_targets_by_repository_identity'" + ).fetchone() + if "repository_identity" in columns and index_exists is not None: return False - migration_sql = next(sql for version, _, sql in MIGRATIONS if version == 16) - for statement in sql_statements(migration_sql): - if statement.startswith("ALTER TABLE workspaces"): - add_column_if_missing( - connection, - "workspaces", - "target_id", - "TEXT REFERENCES security_targets(id)", - ) - continue - if statement.startswith("ALTER TABLE scans"): - add_column_if_missing( - connection, - "scans", - "target_id", - "TEXT REFERENCES security_targets(id)", - ) - continue - statement = statement.replace("CREATE TABLE ", "CREATE TABLE IF NOT EXISTS ", 1) - statement = statement.replace("CREATE INDEX ", "CREATE INDEX IF NOT EXISTS ", 1) - connection.execute(statement) + add_column_if_missing(connection, "security_targets", "repository_identity", "TEXT") connection.execute( - """ - UPDATE scans - SET target_id = NULL - WHERE target_id IS NOT NULL - AND NOT EXISTS ( - SELECT 1 FROM security_targets WHERE security_targets.id = scans.target_id - ) - """ + "CREATE INDEX IF NOT EXISTS security_targets_by_repository_identity " + "ON security_targets(repository_identity)" ) return True diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index 47d2ea6d3..e15813c8a 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -4,16 +4,200 @@ import argparse import hashlib +import os import sqlite3 +import sys from datetime import datetime, timezone from pathlib import Path +# Some plugin hosts launch Python with safe-path isolation enabled. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from filesystem_identity import ( + serialize_filesystem_identity, + stored_filesystem_identity_matches, +) +from workbench_target import git_output + def stable_target_id(target: Path) -> str: digest = hashlib.sha256(f"local-workspace\0{target}".encode()).hexdigest() return f"target_sha256_{digest}" +def repository_relative_path(target: Path) -> str | None: + """Return the target's normalized location within its own Git worktree.""" + worktree = _repository_worktree(target) + return worktree[1] if worktree is not None else None + + +def _repository_worktree(target: Path) -> tuple[Path, str] | None: + worktree_root = git_output(target, "rev-parse", "--show-toplevel") + if worktree_root is None: + return None + try: + canonical_root = Path(os.path.realpath(worktree_root)) + relative = Path(os.path.realpath(target)).relative_to(canonical_root) + except (OSError, ValueError): + return None + return canonical_root, os.path.normcase(os.fspath(relative)).replace(os.sep, "/") + + +def repository_identity(target: Path | str) -> str | None: + """Identify matching Git worktree targets without storing remote credentials.""" + target = Path(target) + common_directory = git_output( + target, "rev-parse", "--path-format=absolute", "--git-common-dir" + ) + if common_directory is None: + return None + worktree = _repository_worktree(target) + if worktree is None: + return None + worktree_root, relative = worktree + registered = git_output(target, "worktree", "list", "--porcelain", "-z") + if registered is None: + return None + canonical_root = os.path.normcase(os.fspath(worktree_root)) + if not any( + os.path.normcase(os.path.realpath(record.removeprefix("worktree "))) + == canonical_root + for record in registered.split("\0") + if record.startswith("worktree ") + ): + return None + canonical_directory = os.path.normcase(os.path.realpath(common_directory)) + try: + metadata = Path(canonical_directory).stat() + except OSError: + return None + device = serialize_filesystem_identity(metadata.st_dev) + inode = serialize_filesystem_identity(metadata.st_ino) + material = f"git-common-dir\0{canonical_directory}\0{device}\0{inode}\0{relative}" + return f"repository_sha256_{hashlib.sha256(material.encode()).hexdigest()}" + + +def _supports_repository_identity(connection: sqlite3.Connection) -> bool: + return any( + row["name"] == "repository_identity" + for row in connection.execute("PRAGMA table_info(security_targets)") + ) + + +def _verified_repository_identity( + connection: sqlite3.Connection, target_id: str, target_path: str +) -> str | None: + target = Path(target_path) + try: + metadata = target.stat() + except OSError: + return None + + scan_columns = { + row["name"] for row in connection.execute("PRAGMA table_info(scans)") + } + if not {"target_id", "target_path"} <= scan_columns: + return None + if not {"target_device", "target_inode"} <= scan_columns: + historical_scan = connection.execute( + "SELECT 1 FROM scans WHERE target_id = ? OR target_path = ? LIMIT 1", + (target_id, target_path), + ).fetchone() + return None if historical_scan is not None else repository_identity(target) + + scans = connection.execute( + """ + SELECT target_device, target_inode + FROM scans + WHERE target_id = ? OR target_path = ? + """, + (target_id, target_path), + ) + if any( + not stored_filesystem_identity_matches(scan["target_device"], metadata.st_dev) + or not stored_filesystem_identity_matches(scan["target_inode"], metadata.st_ino) + for scan in scans + ): + return None + return repository_identity(target) + + +def _require_current_target_owner( + connection: sqlite3.Connection, + target_id: str, + target_path: str, + stored_identity: str | None, +) -> None: + target = Path(target_path) + try: + metadata = target.stat() + except OSError: + return + scan_columns = { + row["name"] for row in connection.execute("PRAGMA table_info(scans)") + } + if not {"target_id", "target_path", "target_device", "target_inode"} <= scan_columns: + return + scans = connection.execute( + """ + SELECT target_device, target_inode + FROM scans + WHERE target_id = ? OR target_path = ? + """, + (target_id, target_path), + ) + historical_scan = False + verified_owner = False + mismatch = False + for scan in scans: + historical_scan = True + device, inode = scan["target_device"], scan["target_inode"] + if device is None and inode is None: + continue + if not stored_filesystem_identity_matches( + device, metadata.st_dev + ) or not stored_filesystem_identity_matches(inode, metadata.st_ino): + mismatch = True + break + verified_owner = True + if ( + mismatch + or stored_identity is not None + and ( + repository_identity(target) != stored_identity + or historical_scan and not verified_owner + ) + ): + raise SystemExit( + f"The repository checkout at {target_path} no longer matches its recorded " + "security scan history; refusing to reuse its target." + ) + + +def backfill_repository_identities(connection: sqlite3.Connection) -> None: + if not _supports_repository_identity(connection): + return + targets = connection.execute( + """ + SELECT id, current_path + FROM security_targets + WHERE repository_identity IS NULL + """ + ).fetchall() + for target in targets: + identity = _verified_repository_identity( + connection, str(target["id"]), target["current_path"] + ) + if identity is not None: + connection.execute( + """ + UPDATE security_targets + SET repository_identity = ? + WHERE id = ? AND repository_identity IS NULL + """, + (identity, target["id"]), + ) + + def backfill_security_targets(connection: sqlite3.Connection) -> None: rows = connection.execute( """ @@ -24,7 +208,7 @@ def backfill_security_targets(connection: sqlite3.Connection) -> None: ).fetchall() for row in rows: target_path = row["target_path"] - target_id = ensure_security_target(connection, target_path) + target_id = ensure_security_target(connection, target_path, verify_ownership=False) connection.execute( "UPDATE workspaces SET target_id = ? WHERE target_path = ? AND target_id IS NULL", (target_id, target_path), @@ -33,25 +217,58 @@ def backfill_security_targets(connection: sqlite3.Connection) -> None: "UPDATE scans SET target_id = ? WHERE target_path = ? AND target_id IS NULL", (target_id, target_path), ) + backfill_repository_identities(connection) -def ensure_security_target(connection: sqlite3.Connection, target_path: str) -> str: +def ensure_security_target( + connection: sqlite3.Connection, target_path: str, *, verify_ownership: bool = True +) -> str: + supports_identity = _supports_repository_identity(connection) existing = connection.execute( - "SELECT id FROM security_targets WHERE current_path = ?", + "SELECT id, repository_identity FROM security_targets WHERE current_path = ?" + if supports_identity + else "SELECT id FROM security_targets WHERE current_path = ?", (target_path,), ).fetchone() if existing is not None: - return str(existing["id"]) + target_id = str(existing["id"]) + if verify_ownership and supports_identity: + _require_current_target_owner( + connection, target_id, target_path, existing["repository_identity"] + ) + if supports_identity and existing["repository_identity"] is None: + identity = _verified_repository_identity(connection, target_id, target_path) + if identity is not None: + connection.execute( + """ + UPDATE security_targets + SET repository_identity = ? + WHERE id = ? AND repository_identity IS NULL + """, + (identity, target_id), + ) + return target_id target_id = stable_target_id(Path(target_path)) timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - connection.execute( - """ - INSERT OR IGNORE INTO security_targets ( - id, current_path, display_name, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?) - """, - (target_id, target_path, Path(target_path).name, timestamp, timestamp), - ) + if supports_identity: + identity = _verified_repository_identity(connection, target_id, target_path) + connection.execute( + """ + INSERT OR IGNORE INTO security_targets ( + id, current_path, display_name, created_at, updated_at, repository_identity + ) VALUES (?, ?, ?, ?, ?, ?) + """, + (target_id, target_path, Path(target_path).name, timestamp, timestamp, identity), + ) + else: + connection.execute( + """ + INSERT OR IGNORE INTO security_targets ( + id, current_path, display_name, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?) + """, + (target_id, target_path, Path(target_path).name, timestamp, timestamp), + ) return target_id diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts index 3fde8e860..92d993612 100644 --- a/sdk/typescript/src/api.ts +++ b/sdk/typescript/src/api.ts @@ -1286,9 +1286,7 @@ export class CodexSecurity { scanId, repository: repo, previousFindings: previousFindings.filter( - (finding) => - finding["scanId"] !== scanId && - finding["targetId"] === targetId, + (finding) => finding["scanId"] !== scanId, ), falsePositives: falsePositiveExamples as Record[], findings: result.findings.findings, diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 7d5a77c60..f751d9130 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -919,19 +919,63 @@ export async function main( ); return presentHistory( await history( - ["list-repositories"], + ["list-scans", "--repository", repository, "--worktrees-only"], async (value): Promise => { - const target = (value["repositories"] as JsonObject[]).find( - (entry) => entry["targetPath"] === repository, + const targetIds = new Set( + (value["scans"] as JsonObject[]) + .map((scan) => scan["targetId"]) + .filter( + (targetId): targetId is string => + typeof targetId === "string", + ), ); - const findings = - target === undefined - ? [] - : await listRepositoryFindings( - dependencies.runWorkbench, - target["targetId"] as string, - ); - return { repository, findings: findings ?? [] }; + const occurrences = new Set(); + const findings: JsonObject[] = []; + for (const targetId of targetIds) { + const targetFindings = await listRepositoryFindings( + dependencies.runWorkbench, + targetId, + ); + for (const finding of targetFindings ?? []) { + const occurrenceId = finding["occurrenceId"]; + if (typeof occurrenceId === "string") { + if (occurrences.has(occurrenceId)) continue; + occurrences.add(occurrenceId); + } + findings.push(finding); + } + } + findings.sort((before, after) => { + const rank = (finding: JsonObject): number => { + const severity = ( + finding["severity"] as JsonObject | undefined + )?.["level"]; + const index = DISPLAY_SEVERITIES.indexOf( + severity as SeverityLevel, + ); + return index === -1 ? DISPLAY_SEVERITIES.length : index; + }; + const severity = rank(before) - rank(after); + if (severity !== 0) return severity; + + const beforeCreated = before["createdAt"]; + const afterCreated = after["createdAt"]; + if ( + typeof beforeCreated === "string" && + typeof afterCreated === "string" && + beforeCreated !== afterCreated + ) { + return beforeCreated < afterCreated ? 1 : -1; + } + + const beforeId = before["occurrenceId"]; + const afterId = after["occurrenceId"]; + if (typeof beforeId !== "string" || typeof afterId !== "string") { + return 0; + } + return beforeId < afterId ? -1 : beforeId > afterId ? 1 : 0; + }); + return { repository, findings }; }, ), "findings", diff --git a/sdk/typescript/tests-ts/cli-workbench.test.ts b/sdk/typescript/tests-ts/cli-workbench.test.ts index 9766a2615..620709ec3 100644 --- a/sdk/typescript/tests-ts/cli-workbench.test.ts +++ b/sdk/typescript/tests-ts/cli-workbench.test.ts @@ -19,8 +19,8 @@ describe("CLI workbench", () => { const calls: Array = []; const responses: JsonObject[] = [ { - repositories: [ - { targetId: "other", targetPath: `${repository}-clone` }, + scans: [ + { targetId: "selected", targetPath: repository }, { targetId: "selected", targetPath: repository }, ], }, @@ -37,7 +37,12 @@ describe("CLI workbench", () => { }), ), ).toBe(0); - expect(calls[0]).toEqual(["list-repositories"]); + expect(calls[0]).toEqual([ + "list-scans", + "--repository", + repository, + "--worktrees-only", + ]); expect(calls[1]).toEqual([ "list-global-findings", "--target-id", @@ -75,6 +80,193 @@ describe("CLI workbench", () => { } }); + test.each([ + { + name: "canonical checkout and linked worktree", + requested: "/repositories/project", + related: "/repositories/project-worktree", + }, + { + name: "linked worktree and canonical checkout", + requested: "/repositories/project-worktree", + related: "/repositories/project", + }, + { + name: "explicitly linked repository alias", + requested: "/repositories/project", + related: "/repositories/linked-alias", + }, + ])( + "combines open findings from the $name", + async ({ requested, related }) => { + const calls: Array = []; + const findingsByTarget: Record = { + requested: [ + { + occurrenceId: "requested-medium", + title: "Requested checkout finding", + severity: { level: "medium" }, + createdAt: "2026-08-13T12:00:00Z", + status: "open", + }, + { + occurrenceId: "duplicate-high", + title: "Duplicated finding", + severity: { level: "high" }, + createdAt: "2026-08-12T12:00:00Z", + status: "open", + }, + ], + related: [ + { + occurrenceId: "related-high", + title: "Related checkout finding", + severity: { level: "high" }, + createdAt: "2026-08-14T12:00:00Z", + status: "open", + }, + { + occurrenceId: "duplicate-high", + title: "Duplicated finding", + severity: { level: "high" }, + createdAt: "2026-08-12T12:00:00Z", + status: "open", + }, + { + occurrenceId: "closed-critical", + title: "Closed finding", + severity: { level: "critical" }, + createdAt: "2026-08-15T12:00:00Z", + status: "closed", + }, + ], + }; + const stdout = capture(); + expect( + await main( + ["findings", "list", requested, "--json"], + stdout.stream, + capture().stream, + dependencies({ + onWorkbench: (args): JsonObject => { + calls.push(args); + if (args[0] === "list-scans") { + return { + scans: [ + { targetId: "requested", targetPath: requested }, + { targetId: "related", targetPath: related }, + { targetId: "requested", targetPath: requested }, + ], + }; + } + const targetId = args[args.indexOf("--target-id") + 1]!; + const requestedStatus = args[args.indexOf("--status") + 1]; + return { + findings: findingsByTarget[targetId]!.filter( + (finding) => finding["status"] === requestedStatus, + ), + nextOffset: null, + }; + }, + }), + ), + ).toBe(0); + expect(calls).toEqual([ + ["list-scans", "--repository", requested, "--worktrees-only"], + [ + "list-global-findings", + "--target-id", + "requested", + "--status", + "open", + ], + ["list-global-findings", "--target-id", "related", "--status", "open"], + ]); + expect(JSON.parse(stdout.text())).toEqual({ + repository: requested, + findings: [ + findingsByTarget["related"]![0], + findingsByTarget["requested"]![1], + findingsByTarget["requested"]![0], + ], + }); + }, + ); + + test("does not query findings for same-origin-only clones", async () => { + const repository = resolve("/current/repository"); + const calls: Array = []; + const stdout = capture(); + const safeFinding: JsonObject = { + occurrenceId: "requested-finding", + title: "Finding in the requested worktree", + status: "open", + }; + expect( + await main( + ["findings", "list", "--json"], + stdout.stream, + capture().stream, + dependencies({ + onWorkbench: (args): JsonObject => { + calls.push(args); + if (args[0] === "list-scans") { + const scans: JsonObject[] = [ + { targetId: "requested", targetPath: repository }, + ]; + if (!args.includes("--worktrees-only")) { + scans.push({ + targetId: "same-origin-clone", + targetPath: `${repository}-clone`, + }); + } + return { scans }; + } + const targetId = args[args.indexOf("--target-id") + 1]; + return { + findings: + targetId === "requested" + ? [safeFinding] + : [{ occurrenceId: "private-clone-finding", status: "open" }], + nextOffset: null, + }; + }, + }), + ), + ).toBe(0); + expect(calls).toEqual([ + ["list-scans", "--repository", repository, "--worktrees-only"], + ["list-global-findings", "--target-id", "requested", "--status", "open"], + ]); + expect(JSON.parse(stdout.text())).toEqual({ + repository, + findings: [safeFinding], + }); + }); + + test("returns no findings when no related repository has been scanned", async () => { + const calls: Array = []; + const stdout = capture(); + const repository = resolve("/current/repository"); + expect( + await main( + ["findings", "list", "--json"], + stdout.stream, + capture().stream, + dependencies({ + onWorkbench: (args) => { + calls.push(args); + return { scans: [] }; + }, + }), + ), + ).toBe(0); + expect(calls).toEqual([ + ["list-scans", "--repository", repository, "--worktrees-only"], + ]); + expect(JSON.parse(stdout.text())).toEqual({ repository, findings: [] }); + }); + test("lists repository and scan-root history without starting Codex", async () => { const repository = resolve("/current/repository"); const cases: Array<[string[], string[]]> = [ diff --git a/sdk/typescript/tests-ts/repository-feedback-aliases.test.ts b/sdk/typescript/tests-ts/repository-feedback-aliases.test.ts new file mode 100644 index 000000000..b5f2e3a19 --- /dev/null +++ b/sdk/typescript/tests-ts/repository-feedback-aliases.test.ts @@ -0,0 +1,200 @@ +import { spawnSync } from "node:child_process"; +import { join } from "node:path"; +import { expect, test } from "bun:test"; +import { PLUGIN_ROOT } from "./plugin-root.js"; + +test("reuses reviewed feedback only across matching persisted repository identities", () => { + const python = (Bun.which("python3") ?? Bun.which("python"))!; + const probe = ` +import json, os, sqlite3, sys, tempfile +sys.path.insert(0, sys.argv[1]) +from workbench_feedback import get_scan_feedback + +scenario = sys.argv[2] +connection = sqlite3.connect(":memory:") +connection.row_factory = sqlite3.Row +connection.executescript(""" +CREATE TABLE security_targets(id TEXT, current_path TEXT, display_name TEXT, origin TEXT); +CREATE TABLE scans(id TEXT, target_id TEXT, status TEXT, completed_at TEXT, started_at TEXT, updated_at TEXT, scope TEXT); +CREATE TABLE findings(id TEXT PRIMARY KEY, fingerprint TEXT, rule_id TEXT, identity_anchor TEXT, identity_instance TEXT); +CREATE TABLE finding_occurrences(id TEXT, finding_id TEXT, scan_id TEXT, title TEXT, summary TEXT, severity TEXT, created_at TEXT); +CREATE TABLE finding_triage(occurrence_id TEXT, status TEXT, close_reason TEXT, note TEXT, updated_at TEXT); +CREATE TABLE finding_locations(id INTEGER PRIMARY KEY, occurrence_id TEXT, relative_path TEXT, start_line INTEGER, end_line INTEGER, role TEXT, sort_order INTEGER); +CREATE TABLE scan_comparison_matches(before_occurrence_id TEXT, after_occurrence_id TEXT); +""") +if scenario == "identities": + connection.execute("ALTER TABLE security_targets ADD COLUMN repository_identity TEXT") + +origin = "https://example.invalid/synthetic/repository" +for target, identity in [ + ("primary", "common-git-directory::."), + ("linked", "common-git-directory::."), + ("same-origin-clone", "independent-git-directory::."), + ("different-scope", "common-git-directory::packages/api"), + ("unknown-first", None), + ("unknown-second", None), +]: + connection.execute( + "INSERT INTO security_targets(id, current_path, display_name, origin) VALUES (?, ?, ?, ?)", + (target, f"/{target}", target, origin), + ) + if scenario == "identities": + connection.execute( + "UPDATE security_targets SET repository_identity = ? WHERE id = ?", + (identity, target), + ) + +def add_scan(scan_id, target, day, status="complete"): + timestamp = f"2026-03-{day:02d}T00:00:00Z" + completed_at = timestamp if status == "complete" else None + connection.execute("INSERT INTO scans VALUES (?, ?, ?, ?, ?, ?, ?)", (scan_id, target, status, completed_at, timestamp, timestamp, "repository")) + +def add_finding(scan_id, finding_id, day, *, status="closed", reason="false_positive", note="Reviewed and safe"): + occurrence_id = f"{scan_id}:{finding_id}" + connection.execute( + "INSERT OR IGNORE INTO findings VALUES (?, ?, ?, ?, ?)", + (finding_id, f"fingerprint-{finding_id}", "synthetic-rule", f"anchor-{finding_id}", None), + ) + connection.execute( + "INSERT INTO finding_occurrences VALUES (?, ?, ?, ?, ?, ?, ?)", + (occurrence_id, finding_id, scan_id, finding_id, "Synthetic summary", "high", f"2026-03-{day:02d}T00:00:00Z"), + ) + connection.execute( + "INSERT INTO finding_locations(occurrence_id, relative_path, start_line, end_line, role, sort_order) VALUES (?, ?, ?, ?, ?, ?)", + (occurrence_id, "src/auth.py", 3, 4, "root_control", 0), + ) + connection.execute( + "INSERT INTO finding_triage VALUES (?, ?, ?, ?, ?)", + (occurrence_id, status, reason, note, f"2026-03-{day:02d}T12:00:00Z"), + ) + +for scan_id, target, day in [ + ("primary-reviewed", "primary", 1), + ("linked-reviewed", "linked", 2), + ("linked-reopened", "linked", 3), + ("clone-reviewed", "same-origin-clone", 4), + ("scope-reviewed", "different-scope", 5), + ("unknown-first-reviewed", "unknown-first", 6), + ("unknown-second-reviewed", "unknown-second", 7), +]: + add_scan(scan_id, target, day) + +add_finding("primary-reviewed", "primary-false-positive", 1) +add_finding("primary-reviewed", "reopened-across-alias", 1) +add_finding("primary-reviewed", "renamed-before-reopening", 1) +add_finding("linked-reviewed", "linked-false-positive", 2) +add_finding("linked-reviewed", "linked-wont-fix", 2, reason="wont_fix") +add_finding("linked-reviewed", "linked-no-note", 2, note=" ") +add_finding("linked-reopened", "reopened-across-alias", 3, status="open", reason=None, note=None) +add_finding("linked-reopened", "renamed-after-reopening", 3, status="open", reason=None, note=None) +connection.execute( + "INSERT INTO scan_comparison_matches VALUES (?, ?)", + ("primary-reviewed:renamed-before-reopening", "linked-reopened:renamed-after-reopening"), +) +add_finding("clone-reviewed", "clone-false-positive", 4) +add_finding("scope-reviewed", "scope-false-positive", 5) +add_finding("unknown-first-reviewed", "unknown-first-false-positive", 6) +add_finding("unknown-second-reviewed", "unknown-second-false-positive", 7) +add_scan("linked-incomplete", "linked", 8, status="running") +add_finding("linked-incomplete", "linked-incomplete-false-positive", 8) + +for target in ["primary", "linked", "same-origin-clone", "different-scope", "unknown-first", "unknown-second"]: + add_scan(f"current-{target}", target, 9, status="running") + +def feedback(target): + scan = connection.execute("SELECT * FROM scans WHERE id = ?", (f"current-{target}",)).fetchone() + return get_scan_feedback(connection, scan) + +result = { + "primary": feedback("primary"), + "linked": feedback("linked"), + "clone": feedback("same-origin-clone"), + "differentScope": feedback("different-scope"), + "unknownFirst": feedback("unknown-first"), + "unknownSecond": feedback("unknown-second"), +} +if scenario == "identities": + connection.execute("ALTER TABLE scans ADD COLUMN target_path TEXT") + connection.execute("ALTER TABLE scans ADD COLUMN target_device INTEGER") + connection.execute("ALTER TABLE scans ADD COLUMN target_inode INTEGER") + with tempfile.TemporaryDirectory() as reused_path: + metadata = os.stat(reused_path) + connection.execute( + "UPDATE security_targets SET current_path = ? WHERE id = ?", + (reused_path, "primary"), + ) + connection.execute( + "UPDATE scans SET target_path = ?, target_device = ?, target_inode = ? WHERE target_id = ?", + (reused_path, metadata.st_dev, metadata.st_ino + 1, "primary"), + ) + result["reusedPath"] = feedback("primary") + result["deletedAlias"] = feedback("linked") +print(json.dumps(result)) +`; + + const run = (scenario: "identities" | "legacy") => { + const execution = spawnSync( + python, + ["-I", "-B", "-c", probe, join(PLUGIN_ROOT, "scripts"), scenario], + { encoding: "utf8", timeout: 10_000 }, + ); + expect(execution.status, execution.stderr).toBe(0); + return JSON.parse(execution.stdout) as Record< + string, + { + scanId: string; + targetId: string; + falsePositives: Array>; + } + >; + }; + + const identities = run("identities"); + expect(identities["primary"]).toMatchObject({ + scanId: "current-primary", + targetId: "primary", + falsePositives: [ + { + findingId: "linked-false-positive", + sourceScanId: "linked-reviewed", + reason: "Reviewed and safe", + }, + { + findingId: "primary-false-positive", + sourceScanId: "primary-reviewed", + }, + ], + }); + expect(identities["linked"]?.falsePositives).toEqual( + identities["primary"]?.falsePositives, + ); + expect(identities["clone"]?.falsePositives).toMatchObject([ + { findingId: "clone-false-positive" }, + ]); + expect(identities["differentScope"]?.falsePositives).toMatchObject([ + { findingId: "scope-false-positive" }, + ]); + expect(identities["unknownFirst"]?.falsePositives).toMatchObject([ + { findingId: "unknown-first-false-positive" }, + ]); + expect(identities["unknownSecond"]?.falsePositives).toMatchObject([ + { findingId: "unknown-second-false-positive" }, + ]); + expect(identities["reusedPath"]?.falsePositives).toEqual([]); + expect(identities["deletedAlias"]?.falsePositives).toEqual( + identities["linked"]?.falsePositives, + ); + + const legacy = run("legacy"); + expect(legacy["primary"]?.falsePositives).toMatchObject([ + { findingId: "reopened-across-alias", sourceScanId: "primary-reviewed" }, + { + findingId: "renamed-before-reopening", + sourceScanId: "primary-reviewed", + }, + { findingId: "primary-false-positive", sourceScanId: "primary-reviewed" }, + ]); + expect(legacy["linked"]?.falsePositives).toMatchObject([ + { findingId: "linked-false-positive", sourceScanId: "linked-reviewed" }, + ]); +}); diff --git a/sdk/typescript/tests-ts/repository-findings-worktrees.test.ts b/sdk/typescript/tests-ts/repository-findings-worktrees.test.ts new file mode 100644 index 000000000..f231e3aa7 --- /dev/null +++ b/sdk/typescript/tests-ts/repository-findings-worktrees.test.ts @@ -0,0 +1,177 @@ +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { afterEach, describe, expect, test } from "bun:test"; +import { CodexSecurity } from "../src/index.js"; +import { + completedEvents, + createApiTestFixtures, + preparedRuntime, +} from "./support/api-events.js"; + +const fixtures = createApiTestFixtures(); +const TestClient = CodexSecurity as unknown as new ( + config: Record, + dependencies: Record, +) => CodexSecurity; + +afterEach(fixtures.cleanup); + +describe("repository findings across linked worktrees", () => { + test("matches only repository-identity-scoped history from another worktree", async () => { + const root = await fixtures.temporaryDirectory(); + const repository = join(root, "repository"); + const codexHome = join(root, "codex-home"); + const scanDir = join(root, "scan"); + await mkdir(repository); + await mkdir(codexHome); + await mkdir(scanDir, { mode: 0o700 }); + + const targetId = "target_sha256_example"; + const linkedFinding = { + findingId: "linked_worktree_finding", + occurrenceId: "linked_worktree_occurrence", + scanId: "linked_worktree_scan", + targetId: "target_linked_worktree", + }; + const unrelatedFinding = { + findingId: "independent_clone_finding", + occurrenceId: "independent_clone_occurrence", + scanId: "independent_clone_scan", + targetId: "target_independent_clone", + }; + const currentFinding = { + findingId: "csf_852f90d6e1177502ff113d4a", + occurrenceId: "occ_e79cb19591e696572a1c22be", + scanId: "scan_example_001", + targetId, + }; + const mergedFinding = { + ...linkedFinding, + title: "Unsafe archive extraction", + summary: "Archive entries can escape their destination.", + severity: { level: "high" as const }, + status: "open" as const, + confirmedInLatestScan: true, + knownScanIds: [linkedFinding.scanId, currentFinding.scanId], + }; + const commands: Array = []; + const matchedInputs: Array<{ + before: readonly Record[]; + after: readonly Record[]; + }> = []; + + const client = new TestClient( + {}, + { + environment: {}, + prepareRuntime: async () => preparedRuntime(codexHome), + resolvePluginPython: async () => "/managed/python", + prepareOutputDir: async () => scanDir, + repositoryRevision: async () => "deadbeef", + runWorkbench: async (_options: unknown, args: readonly string[]) => { + commands.push(args); + if (args[0] === "register-cli-scan") { + return { + scanId: currentFinding.scanId, + targetId, + targetRevision: "deadbeef", + scanDir: args[args.indexOf("--scan-dir") + 1], + contract: { target: { allowedKinds: ["git_revision"] } }, + }; + } + if (args[0] === "get-scan-feedback") { + return { + scanId: currentFinding.scanId, + targetId, + falsePositives: [], + }; + } + if (args[0] === "list-global-findings") { + expect(args).toContain("--target-id"); + expect(args).toContain(targetId); + return { + findings: args.includes("--status") + ? [mergedFinding] + : [linkedFinding, currentFinding], + }; + } + if (args[0] === "list-unmatched-scan-pairs") { + return { + batches: [ + { + afterScanId: currentFinding.scanId, + afterFindings: [currentFinding], + beforeScans: [ + { + scanId: linkedFinding.scanId, + findings: [linkedFinding], + }, + { + scanId: unrelatedFinding.scanId, + findings: [unrelatedFinding], + }, + ], + }, + ], + }; + } + return {}; + }, + async matchFindings(input: (typeof matchedInputs)[number]) { + matchedInputs.push(input); + return { + matches: [ + { + beforeOccurrenceIds: [linkedFinding.occurrenceId], + afterOccurrenceIds: [currentFinding.occurrenceId], + confidence: "high", + reason: "The linked worktree has the same root cause.", + }, + ], + uncertain: [], + }; + }, + createCodex: () => ({ + startThread: () => ({ + id: null, + async runStreamed() { + await fixtures.copyCompletedScan(root); + return { events: completedEvents() }; + }, + }), + }), + }, + ); + + const result = await client.run(repository); + + expect(matchedInputs).toEqual([ + { before: [linkedFinding], after: [currentFinding] }, + ]); + expect( + commands.filter(([command]) => command === "save-scan-comparison"), + ).toEqual([ + [ + "save-scan-comparison", + "--before-scan-id", + linkedFinding.scanId, + "--after-scan-id", + currentFinding.scanId, + "--matches-json", + JSON.stringify({ + matches: [ + { + beforeOccurrenceIds: [linkedFinding.occurrenceId], + afterOccurrenceIds: [currentFinding.occurrenceId], + confidence: "high", + reason: "The linked worktree has the same root cause.", + }, + ], + uncertain: [], + }), + ], + ]); + expect(result.repositoryFindings).toEqual([mergedFinding]); + await client.close(); + }); +}); diff --git a/sdk/typescript/tests-ts/repository-findings.test.ts b/sdk/typescript/tests-ts/repository-findings.test.ts index 1a001995c..7a49c8171 100644 --- a/sdk/typescript/tests-ts/repository-findings.test.ts +++ b/sdk/typescript/tests-ts/repository-findings.test.ts @@ -114,3 +114,230 @@ print(json.dumps(result)) "renamed-again", ]); }); + +test("shares findings only between explicitly identified repository and scope aliases", () => { + const python = (Bun.which("python3") ?? Bun.which("python"))!; + const probe = ` +import argparse, json, os, sqlite3, sys, tempfile +sys.path.insert(0, sys.argv[1]) +import workbench_native_indexes as indexes + +connection = sqlite3.connect(":memory:") +connection.row_factory = sqlite3.Row +connection.executescript(""" +CREATE TABLE security_targets(id TEXT, current_path TEXT, display_name TEXT, repository_identity TEXT, origin TEXT); +CREATE TABLE scans(id TEXT, target_id TEXT, scope TEXT, updated_at TEXT, status TEXT, started_at TEXT); +CREATE TABLE finding_occurrences(id TEXT, finding_id TEXT, severity TEXT, created_at TEXT, scan_id TEXT, title TEXT, summary TEXT); +CREATE TABLE finding_triage(occurrence_id TEXT, status TEXT, updated_at TEXT, close_reason TEXT); +CREATE TABLE finding_locations(occurrence_id TEXT, relative_path TEXT, role TEXT, sort_order INTEGER); +CREATE TABLE scan_comparison_matches(before_occurrence_id TEXT, after_occurrence_id TEXT); +""") +origin = "https://example.invalid/synthetic/repository" +connection.executemany("INSERT INTO security_targets VALUES (?, ?, ?, ?, ?)", [ + ("primary", "/primary", "Primary", "common-git-directory::.", origin), + ("linked", "/linked", "Linked", "common-git-directory::.", origin), + ("same-origin-clone", "/clone", "Clone", "independent-git-directory::.", origin), + ("different-scope", "/primary/packages/api", "Scoped", "common-git-directory::packages/api", origin), + ("unknown-first", "/unknown-first", "Unknown first", None, origin), + ("unknown-second", "/unknown-second", "Unknown second", None, origin), +]) + +def add_scan(scan_id, target, day): + timestamp = f"2026-02-{day:02d}T00:00:00Z" + connection.execute("INSERT INTO scans VALUES (?, ?, ?, ?, ?, ?)", (scan_id, target, "repository", timestamp, "complete", timestamp)) + +def add_finding(occurrence, finding, scan, severity="high"): + started = connection.execute("SELECT started_at FROM scans WHERE id = ?", (scan,)).fetchone()[0] + connection.execute("INSERT INTO finding_occurrences VALUES (?, ?, ?, ?, ?, ?, ?)", (occurrence, finding, severity, started, scan, finding, "Summary")) + connection.execute("INSERT INTO finding_locations VALUES (?, ?, ?, ?)", (occurrence, "src/auth.py", "root_control", 0)) + +for scan_id, target, day in [ + ("primary-old", "primary", 1), + ("linked-reviewed", "linked", 2), + ("primary-open", "primary", 3), + ("linked-latest", "linked", 4), + ("clone-scan", "same-origin-clone", 5), + ("scope-scan", "different-scope", 6), + ("unknown-first-scan", "unknown-first", 7), + ("unknown-second-scan", "unknown-second", 8), +]: + add_scan(scan_id, target, day) + +for occurrence, finding, scan in [ + ("dismissed-primary", "dismissed-original", "primary-old"), + ("dismissed-linked", "dismissed-renamed", "linked-reviewed"), + ("fixed-primary", "fixed-original", "primary-old"), + ("fixed-linked", "fixed-rediscovered", "linked-reviewed"), + ("wont-fix-primary", "wont-fix-original", "primary-old"), + ("wont-fix-linked", "wont-fix-renamed", "linked-reviewed"), + ("open-primary", "primary-only", "primary-open"), + ("same-id-primary", "same-id", "primary-open"), + ("open-linked", "linked-only", "linked-latest"), + ("same-id-linked", "same-id", "linked-latest"), + ("clone-occurrence", "clone-only", "clone-scan"), + ("scope-occurrence", "scope-only", "scope-scan"), + ("unknown-first-occurrence", "unknown-first-only", "unknown-first-scan"), + ("unknown-second-occurrence", "unknown-second-only", "unknown-second-scan"), +]: + add_finding(occurrence, finding, scan) + +connection.executemany("INSERT INTO scan_comparison_matches VALUES (?, ?)", [ + ("dismissed-primary", "dismissed-linked"), + ("fixed-primary", "fixed-linked"), + ("wont-fix-primary", "wont-fix-linked"), + ("open-linked", "clone-occurrence"), + ("open-linked", "scope-occurrence"), + ("unknown-first-occurrence", "unknown-second-occurrence"), +]) +connection.executemany("INSERT INTO finding_triage VALUES (?, ?, ?, ?)", [ + ("dismissed-primary", "closed", "2026-02-01T12:00:00Z", "false_positive"), + ("fixed-primary", "closed", "2026-02-01T12:00:00Z", "already_fixed"), + ("wont-fix-primary", "closed", "2026-02-01T12:00:00Z", "wont_fix"), +]) + +def findings(target, status="open", limit=20, offset=0): + arguments = argparse.Namespace(limit=limit, offset=offset, query=None, severity=None, status=status, target_id=target) + return indexes.list_global_findings(connection, arguments) + +result = { + "primary": findings("primary"), + "linked": findings("linked"), + "closed": findings("primary", "closed"), + "all": findings("primary", None), + "clone": findings("same-origin-clone"), + "differentScope": findings("different-scope"), + "unknownFirst": findings("unknown-first"), + "unknownSecond": findings("unknown-second"), + "firstPage": findings("primary", limit=2), + "secondPage": findings("primary", limit=2, offset=2), + "thirdPage": findings("primary", limit=2, offset=4), +} +connection.execute( + "INSERT INTO finding_triage VALUES (?, ?, ?, ?)", + ("same-id-primary", "closed", "2026-02-05T00:00:00Z", "false_positive"), +) +result["sameIdDismissed"] = findings("primary") +result["sameIdClosed"] = findings("linked", "closed") +connection.execute( + "INSERT INTO finding_triage VALUES (?, ?, ?, ?)", + ("same-id-linked", "open", "2026-02-06T00:00:00Z", None), +) +result["sameIdReopened"] = findings("primary") +connection.execute("ALTER TABLE scans ADD COLUMN target_path TEXT") +connection.execute("ALTER TABLE scans ADD COLUMN target_device INTEGER") +connection.execute("ALTER TABLE scans ADD COLUMN target_inode INTEGER") +with tempfile.TemporaryDirectory() as reused_path: + metadata = os.stat(reused_path) + connection.execute( + "UPDATE security_targets SET current_path = ? WHERE id = ?", + (reused_path, "primary"), + ) + connection.execute( + "UPDATE scans SET target_path = ?, target_device = ?, target_inode = ? WHERE target_id = ?", + (reused_path, metadata.st_dev, metadata.st_ino + 1, "primary"), + ) + result["reusedPath"] = findings("primary") + result["deletedAlias"] = findings("linked") +print(json.dumps(result)) +`; + + const execution = spawnSync( + python, + ["-I", "-B", "-c", probe, join(PLUGIN_ROOT, "scripts")], + { encoding: "utf8", timeout: 10_000 }, + ); + expect(execution.status, execution.stderr).toBe(0); + + const result = JSON.parse(execution.stdout) as Record< + string, + { + findings: Array>; + nextOffset: number | null; + } + >; + const primary = result["primary"]!.findings; + expect(result["linked"]!.findings).toEqual(primary); + expect(primary.map((finding) => finding["findingId"])).toEqual([ + "linked-only", + "same-id", + "primary-only", + "fixed-rediscovered", + ]); + expect( + primary.filter((finding) => finding["findingId"] === "same-id"), + ).toMatchObject([ + { + targetId: "linked", + occurrenceCount: 2, + knownScanIds: ["primary-open", "linked-latest"], + matchedFindingIds: ["same-id"], + }, + ]); + expect( + primary.find((finding) => finding["findingId"] === "linked-only"), + ).toMatchObject({ + confirmedInLatestScan: true, + matchedFindingIds: ["linked-only"], + }); + expect( + primary.find((finding) => finding["findingId"] === "primary-only"), + ).toMatchObject({ confirmedInLatestScan: false }); + expect( + primary.find((finding) => finding["findingId"] === "fixed-rediscovered"), + ).toMatchObject({ + status: "open", + knownScanIds: ["primary-old", "linked-reviewed"], + matchedFindingIds: ["fixed-original", "fixed-rediscovered"], + occurrenceCount: 2, + }); + expect(result["closed"]!.findings).toMatchObject([ + { + findingId: "dismissed-renamed", + status: "closed", + matchedFindingIds: ["dismissed-original", "dismissed-renamed"], + }, + { + findingId: "wont-fix-renamed", + status: "closed", + matchedFindingIds: ["wont-fix-original", "wont-fix-renamed"], + }, + ]); + expect(result["all"]!.findings).toHaveLength(6); + expect(result["clone"]!.findings).toMatchObject([ + { findingId: "clone-only", targetId: "same-origin-clone" }, + ]); + expect(result["differentScope"]!.findings).toMatchObject([ + { findingId: "scope-only", targetId: "different-scope" }, + ]); + expect(result["unknownFirst"]!.findings).toMatchObject([ + { findingId: "unknown-first-only", targetId: "unknown-first" }, + ]); + expect(result["unknownSecond"]!.findings).toMatchObject([ + { findingId: "unknown-second-only", targetId: "unknown-second" }, + ]); + expect(result["firstPage"]!.nextOffset).toBe(2); + expect(result["secondPage"]!.nextOffset).toBeNull(); + expect(result["thirdPage"]!.nextOffset).toBeNull(); + expect([ + ...result["firstPage"]!.findings, + ...result["secondPage"]!.findings, + ...result["thirdPage"]!.findings, + ]).toEqual(primary); + expect( + result["sameIdDismissed"]!.findings.some( + (finding) => finding["findingId"] === "same-id", + ), + ).toBe(false); + expect( + result["sameIdClosed"]!.findings.find( + (finding) => finding["findingId"] === "same-id", + ), + ).toMatchObject({ status: "closed", occurrenceCount: 2 }); + expect( + result["sameIdReopened"]!.findings.find( + (finding) => finding["findingId"] === "same-id", + ), + ).toMatchObject({ status: "open", occurrenceCount: 2 }); + expect(result["reusedPath"]!.findings).toEqual([]); + expect(result["deletedAlias"]!.findings).toHaveLength(4); +}); diff --git a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts new file mode 100644 index 000000000..f53e141e1 --- /dev/null +++ b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts @@ -0,0 +1,639 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, test } from "bun:test"; +import { PLUGIN_ROOT } from "./plugin-root.js"; + +const temporaryRoots: string[] = []; +const remote = + "https://fixture-user:SYNTHETIC_PASSWORD@example.test/acme/project.git"; + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } +}); + +function git(repository: string, ...args: string[]): string { + return execFileSync( + "git", + [ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.test", + ...args, + ], + { cwd: repository, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }, + ).trim(); +} + +function fixture(): { + root: string; + repository: string; + worktree: string; + clone: string; +} { + const root = realpathSync( + mkdtempSync(join(tmpdir(), "codex-security-repository-identity-")), + ); + temporaryRoots.push(root); + const repository = join(root, "repository"); + const worktree = join(root, "linked-worktree"); + const clone = join(root, "same-origin-clone"); + mkdirSync(repository); + git(repository, "init", "-q"); + for (const service of ["service-a", "service-b"]) { + mkdirSync(join(repository, service)); + writeFileSync(join(repository, service, "service.py"), "value = 1\n"); + } + git(repository, "add", "."); + git(repository, "commit", "-qm", "fixture"); + git(repository, "remote", "add", "origin", remote); + git(repository, "worktree", "add", "--detach", "-q", worktree, "HEAD"); + execFileSync("git", ["clone", "-q", repository, clone], { + stdio: ["ignore", "pipe", "pipe"], + }); + git(clone, "remote", "set-url", "origin", remote); + return { root, repository, worktree, clone }; +} + +const probe = String.raw` +import argparse +import hashlib +import json +import os +import shutil +import sqlite3 +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, sys.argv[1]) + +import workbench_scan_history as history +from filesystem_identity import serialize_filesystem_identity +from workbench_schema import MIGRATIONS, apply_migrations +from workbench_target_state import ( + backfill_repository_identities, + backfill_security_targets, + ensure_security_target, + repository_identity, + stable_target_id, +) + +scenario = sys.argv[2] +root, repository, worktree, clone = map(Path, sys.argv[3:7]) +timestamp = "2026-08-01T00:00:00Z" +connection = sqlite3.connect(":memory:") +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +apply_migrations(connection, MIGRATIONS, lambda: timestamp, backfill_security_targets) + + +def git(target, *args): + return subprocess.run( + [ + "git", + "-c", "user.name=Fixture", + "-c", "user.email=fixture@example.test", + "-C", str(target), + *args, + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def add_scan(scan_id, target, ownership="current", verify_ownership=True): + target = Path(target) + target_id = ensure_security_target( + connection, str(target), verify_ownership=verify_ownership + ) + metadata = target.stat() + device = serialize_filesystem_identity(metadata.st_dev) + inode = serialize_filesystem_identity(metadata.st_ino) + if ownership == "missing": + device, inode = None, None + elif ownership == "malformed": + inode = None + elif ownership == "mismatch": + inode = serialize_filesystem_identity(metadata.st_ino + 1) + + workspace_id = f"workspace-{scan_id}" + connection.execute( + "INSERT INTO workspaces (id, target_path, target_id, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (workspace_id, str(target), target_id, timestamp, timestamp), + ) + connection.execute( + "INSERT INTO scans (id, workspace_id, target_path, target_id, target_device, " + "target_inode, target_revision, scope, mode, scan_dir, status, phase, " + "started_at, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + scan_id, workspace_id, str(target), target_id, device, inode, + "synthetic-revision", ".", "standard", str(root / "scans" / scan_id), + "complete", "reporting", timestamp, timestamp, timestamp, + ), + ) + connection.execute( + "INSERT INTO scan_progress (scan_id, updated_at) VALUES (?, ?)", + (scan_id, timestamp), + ) + return target_id + + +def listed(target, strict=True): + arguments = argparse.Namespace( + repository=str(target), + worktrees_only=strict, + scan_root=None, + target_id=None, + mode=None, + status=None, + query=None, + limit=None, + offset=0, + ) + return sorted(scan["scanId"] for scan in history.list_scans(connection, arguments)["scans"]) + + +def forged_worktree(): + forged = root / "forged-git-pointer" + forged.mkdir() + common = git(repository, "rev-parse", "--path-format=absolute", "--git-common-dir") + (forged / ".git").write_text(f"gitdir: {common}\n") + return forged + + +if scenario == "identity": + targets = { + "repository": repository, + "worktree": worktree, + "serviceA": repository / "service-a", + "worktreeServiceA": worktree / "service-a", + "serviceB": repository / "service-b", + "clone": clone, + } + target_ids = { + name: ensure_security_target(connection, str(target)) + for name, target in targets.items() + } + identities = { + name: repository_identity(target) + for name, target in targets.items() + } + common = os.path.normcase(os.path.realpath(git( + repository, "rev-parse", "--path-format=absolute", "--git-common-dir" + ))) + metadata = Path(common).stat() + device = serialize_filesystem_identity(metadata.st_dev) + inode = serialize_filesystem_identity(metadata.st_ino) + expected = "repository_sha256_" + hashlib.sha256( + f"git-common-dir\0{common}\0{device}\0{inode}\0.".encode() + ).hexdigest() + forged_identity = repository_identity(forged_worktree()) + git( + repository, + "remote", "set-url", "origin", + "https://different-user:DIFFERENT_SYNTHETIC_PASSWORD@example.test/other/repo.git", + ) + print(json.dumps({ + "identities": identities, + "expected": expected, + "forgedIdentity": forged_identity, + "remoteIndependent": repository_identity(repository) == identities["repository"], + "targetIdsPreserved": all( + target_ids[name] == stable_target_id(target) + for name, target in targets.items() + ), + "stored": { + row["current_path"]: row["repository_identity"] + for row in connection.execute( + "SELECT current_path, repository_identity FROM security_targets" + ) + }, + })) + +elif scenario == "history": + add_scan("canonical-root", repository) + add_scan("linked-root", worktree) + add_scan("spoof-root", clone) + add_scan("canonical-a", repository / "service-a") + add_scan("linked-a", worktree / "service-a") + add_scan("canonical-b", repository / "service-b") + add_scan("spoof-a", clone / "service-a") + add_scan("forged-root", forged_worktree()) + before = { + "strictRoot": listed(repository), + "legacyRoot": listed(repository, strict=False), + "strictServiceA": listed(repository / "service-a"), + "legacyServiceA": listed(repository / "service-a", strict=False), + "strictServiceB": listed(repository / "service-b"), + } + git(repository, "worktree", "remove", "--force", str(worktree)) + after = { + "strictRoot": listed(repository), + "legacyRoot": listed(repository, strict=False), + "strictServiceA": listed(repository / "service-a"), + } + arguments = argparse.Namespace(repository=str(repository), force=False) + matched = history.list_unmatched_scan_pairs( + connection, + arguments, + backfill_finding_details=lambda _connection, _scan: None, + read_coverage=lambda _scan: {}, + ) + + def unavailable(scan): + if scan["id"] == "linked-root": + raise SystemExit("Saved scan artifacts are unavailable.") + return {} + + unavailable_matches = history.list_unmatched_scan_pairs( + connection, + arguments, + backfill_finding_details=lambda _connection, _scan: None, + read_coverage=unavailable, + ) + + def require_scan(database, scan_id): + return database.execute("SELECT * FROM scans WHERE id = ?", (scan_id,)).fetchone() + + comparison_args = argparse.Namespace( + before_scan_id="canonical-root", + after_scan_id="linked-root", + matches_json=json.dumps({"matches": [], "uncertain": []}), + ) + compared = history.compare_scans( + connection, + comparison_args, + require_scan=require_scan, + read_coverage=lambda _scan: {"completeness": "complete"}, + ) + connection.commit() + saved = history.save_scan_comparison( + connection, + comparison_args, + now=lambda: timestamp, + require_scan=require_scan, + read_coverage=lambda _scan: {"completeness": "complete"}, + ) + explicit_clone = history.compare_scans( + connection, + argparse.Namespace(before_scan_id="canonical-root", after_scan_id="spoof-root"), + require_scan=require_scan, + read_coverage=lambda _scan: {"completeness": "complete"}, + ) + + print(json.dumps({ + "before": before, + "after": after, + "matchedScanCount": matched["scanCount"], + "matchingBatches": len(matched["batches"]), + "unavailableScans": unavailable_matches["unavailableScans"], + "compared": compared["beforeScanId"] == "canonical-root" + and compared["afterScanId"] == "linked-root", + "saved": saved["beforeScanId"] == "canonical-root" + and saved["afterScanId"] == "linked-root", + "explicitCloneCompared": explicit_clone["afterScanId"] == "spoof-root", + })) + +elif scenario == "backfill": + add_scan("valid", repository) + add_scan("reused", clone, "mismatch") + add_scan("mixed-current", repository / "service-a") + add_scan("mixed-previous", repository / "service-a", "mismatch") + add_scan("malformed", repository / "service-b", "malformed") + add_scan("missing", worktree) + missing_path = str(worktree) + git(repository, "worktree", "remove", "--force", missing_path) + connection.execute("UPDATE security_targets SET repository_identity = NULL") + before_ids = { + row["current_path"]: row["id"] + for row in connection.execute("SELECT id, current_path FROM security_targets") + } + backfill_repository_identities(connection) + rows = { + row["current_path"]: {"id": row["id"], "identity": row["repository_identity"]} + for row in connection.execute( + "SELECT id, current_path, repository_identity FROM security_targets" + ) + } + print(json.dumps({ + "valid": rows[str(repository)]["identity"], + "reused": rows[str(clone)]["identity"], + "mixed": rows[str(repository / "service-a")]["identity"], + "malformed": rows[str(repository / "service-b")]["identity"], + "missing": rows[missing_path]["identity"], + "idsPreserved": all(rows[path]["id"] == target_id for path, target_id in before_ids.items()), + })) + +elif scenario == "replacement": + add_scan("previous-owner", worktree) + add_scan("trusted-alias", repository) + git(repository, "worktree", "remove", "--force", str(worktree)) + worktree.mkdir() + git(worktree, "init", "-q") + (worktree / "replacement.py").write_text("value = 2\n") + git(worktree, "add", ".") + git(worktree, "commit", "-qm", "replacement") + git(worktree, "remote", "add", "origin", sys.argv[7]) + before = listed(worktree) + regular_before = listed(worktree, strict=False) + try: + add_scan("replacement-owner", worktree) + except SystemExit as error: + registration_error = str(error) + else: + registration_error = None + try: + history.list_unmatched_scan_pairs( + connection, + argparse.Namespace(repository=str(worktree), force=False), + backfill_finding_details=lambda _connection, _scan: None, + read_coverage=lambda _scan: {}, + ) + except SystemExit as error: + matching_error = str(error) + else: + matching_error = None + after = listed(worktree) + + empty_target = clone / "service-a" + ensure_security_target(connection, str(empty_target)) + connection.execute( + "UPDATE security_targets SET repository_identity = ? WHERE current_path = ?", + (repository_identity(repository), str(empty_target)), + ) + try: + ensure_security_target(connection, str(empty_target)) + except SystemExit as error: + empty_target_error = str(error) + else: + empty_target_error = None + + explicit_clone_id = add_scan("explicit-clone", clone) + connection.execute( + "UPDATE security_targets SET repository_identity = ? WHERE id = ?", + (repository_identity(repository), explicit_clone_id), + ) + try: + ensure_security_target(connection, str(clone)) + except SystemExit as error: + explicit_clone_error = str(error) + else: + explicit_clone_error = None + + unverified_target = clone / "service-b" + add_scan("unverified-owner", unverified_target, "missing") + try: + ensure_security_target(connection, str(unverified_target)) + except SystemExit as error: + unverified_owner_error = str(error) + else: + unverified_owner_error = None + print(json.dumps({ + "before": before, + "regularBefore": regular_before, + "after": after, + "registrationError": registration_error, + "matchingError": matching_error, + "emptyTargetError": empty_target_error, + "explicitCloneError": explicit_clone_error, + "unverifiedOwnerError": unverified_owner_error, + "replacementScanCreated": connection.execute( + "SELECT 1 FROM scans WHERE id = 'replacement-owner'" + ).fetchone() is not None, + })) + +elif scenario == "git-replacement": + add_scan("original-repository", repository) + add_scan("original-alias", worktree) + original_identity = repository_identity(repository) + original_metadata = repository.stat() + shutil.rmtree(repository / ".git") + git(repository, "init", "-q") + git(repository, "add", ".") + git(repository, "commit", "-qm", "replacement") + replacement_identity = repository_identity(repository) + try: + ensure_security_target(connection, str(repository)) + except SystemExit as error: + registration_error = str(error) + else: + registration_error = None + replacement_metadata = repository.stat() + print(json.dumps({ + "originalIdentity": original_identity, + "replacementIdentity": replacement_identity, + "checkoutOwnerUnchanged": ( + original_metadata.st_dev == replacement_metadata.st_dev + and original_metadata.st_ino == replacement_metadata.st_ino + ), + "visibleScans": listed(repository), + "regularScans": listed(repository, strict=False), + "registrationError": registration_error, + })) + +elif scenario == "unverified-owner": + ensure_security_target(connection, str(repository)) + add_scan("trusted-alias", worktree) + git(repository, "worktree", "remove", "--force", str(worktree)) + before = listed(repository) + regular_before = listed(repository, strict=False) + add_scan("unverified-owner", repository, "missing") + after = listed(repository) + regular_after = listed(repository, strict=False) + print(json.dumps({ + "before": before, + "regularBefore": regular_before, + "after": after, + "regularAfter": regular_after, + })) + +elif scenario == "nongit": + first = root / "plain-a" + second = root / "plain-b" + first.mkdir() + second.mkdir() + add_scan("plain-a", first) + add_scan("legacy-a", first, "missing") + add_scan("malformed-a", first, "malformed", verify_ownership=False) + add_scan("mismatched-a", first, "mismatch", verify_ownership=False) + add_scan("plain-b", second) + before = connection.execute( + "SELECT NULL AS target_id, ? AS target_path", (str(first),) + ).fetchone() + after = connection.execute( + "SELECT NULL AS target_id, ? AS target_path", (str(second),) + ).fetchone() + same = connection.execute( + "SELECT NULL AS target_id, ? AS target_path", (str(first),) + ).fetchone() + print(json.dumps({ + "firstIdentity": repository_identity(first), + "secondIdentity": repository_identity(second), + "differentNullIdsMatch": history._same_repository(before, after), + "sameNullPathMatches": history._same_repository(before, same), + "firstScans": listed(first), + "secondScans": listed(second), + })) +`; + +function runProbe( + scenario: string, + repositories: ReturnType, +): Record { + const python = Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py"); + expect(python).not.toBeNull(); + if (python === null) throw new Error("A Python interpreter is required."); + const execution = spawnSync( + python, + [ + "-I", + "-B", + "-c", + probe, + join(PLUGIN_ROOT, "scripts"), + scenario, + repositories.root, + repositories.repository, + repositories.worktree, + repositories.clone, + remote, + ], + { encoding: "utf8", timeout: 20_000 }, + ); + expect(execution.status, execution.stderr).toBe(0); + expect(execution.stderr).toBe(""); + return JSON.parse(execution.stdout) as Record; +} + +describe("durable workbench repository identities", () => { + test("hashes Git common directories and repository-relative target scopes", () => { + const repositories = fixture(); + const result = runProbe("identity", repositories); + const identities = result["identities"] as Record; + + expect(identities["repository"]).toBe(result["expected"] as string); + expect(identities["worktree"]).toBe(identities["repository"]); + expect(identities["worktreeServiceA"]).toBe(identities["serviceA"]); + expect(identities["serviceA"]).not.toBe(identities["repository"]); + expect(identities["serviceA"]).not.toBe(identities["serviceB"]); + expect(identities["clone"]).not.toBe(identities["repository"]); + expect(result["forgedIdentity"]).toBeNull(); + expect(result["remoteIndependent"]).toBe(true); + expect(result["targetIdsPreserved"]).toBe(true); + expect(JSON.stringify(result["stored"])).not.toContain( + "SYNTHETIC_PASSWORD", + ); + }, 30_000); + + test("retains removed worktrees while excluding spoofed origins and unrelated scopes", () => { + const result = runProbe("history", fixture()); + const before = result["before"] as Record; + const after = result["after"] as Record; + + expect(before["strictRoot"]).toEqual(["canonical-root", "linked-root"]); + expect(before["legacyRoot"]).toEqual([ + "canonical-root", + "linked-root", + "spoof-root", + ]); + expect(before["strictServiceA"]).toEqual(["canonical-a", "linked-a"]); + expect(before["legacyServiceA"]).toEqual([ + "canonical-a", + "linked-a", + "spoof-a", + ]); + expect(before["strictServiceB"]).toEqual(["canonical-b"]); + expect(after["strictRoot"]).toEqual(["canonical-root", "linked-root"]); + expect(after["legacyRoot"]).toEqual([ + "canonical-root", + "linked-root", + "spoof-root", + ]); + expect(after["strictServiceA"]).toEqual(["canonical-a", "linked-a"]); + expect(result["matchedScanCount"]).toBe(3); + expect(result["matchingBatches"]).toBe(2); + expect(result["unavailableScans"]).toBe(1); + expect(result["compared"]).toBe(true); + expect(result["saved"]).toBe(true); + expect(result["explicitCloneCompared"]).toBe(true); + }, 30_000); + + test("backfills only current owners and preserves every existing target ID", () => { + const result = runProbe("backfill", fixture()); + + expect(result["valid"]).toMatch(/^repository_sha256_[a-f0-9]{64}$/); + expect(result["reused"]).toBeNull(); + expect(result["mixed"]).toBeNull(); + expect(result["malformed"]).toBeNull(); + expect(result["missing"]).toBeNull(); + expect(result["idsPreserved"]).toBe(true); + }, 30_000); + + test("does not expose a previous checkout owner or its trusted aliases", () => { + const result = runProbe("replacement", fixture()); + + expect(result["before"]).toEqual([]); + expect(result["regularBefore"]).toEqual([]); + expect(result["after"]).toEqual([]); + expect(result["registrationError"]).toContain( + "refusing to reuse its target", + ); + expect(result["matchingError"]).toContain("refusing to reuse its target"); + expect(result["emptyTargetError"]).toContain( + "refusing to reuse its target", + ); + expect(result["replacementScanCreated"]).toBe(false); + expect(result["explicitCloneError"]).toContain( + "refusing to reuse its target", + ); + expect(result["unverifiedOwnerError"]).toContain( + "refusing to reuse its target", + ); + }, 30_000); + + test("rejects an in-place Git directory replacement under the same checkout", () => { + const result = runProbe("git-replacement", fixture()); + + expect(result["checkoutOwnerUnchanged"]).toBe(true); + expect(result["replacementIdentity"]).not.toBe(result["originalIdentity"]); + expect(result["visibleScans"]).toEqual([]); + expect(result["regularScans"]).toEqual([]); + expect(result["registrationError"]).toContain( + "refusing to reuse its target", + ); + }, 30_000); + + test("does not expand trusted aliases when historical checkout ownership is unverified", () => { + const result = runProbe("unverified-owner", fixture()); + + expect(result["before"]).toEqual(["trusted-alias"]); + expect(result["regularBefore"]).toEqual(["trusted-alias"]); + expect(result["after"]).toEqual(["unverified-owner"]); + expect(result["regularAfter"]).toEqual(["unverified-owner"]); + }, 30_000); + + test("keeps non-Git paths isolated and never equates unrelated null target IDs", () => { + const result = runProbe("nongit", fixture()); + + expect(result["firstIdentity"]).toBeNull(); + expect(result["secondIdentity"]).toBeNull(); + expect(result["differentNullIdsMatch"]).toBe(false); + expect(result["sameNullPathMatches"]).toBe(true); + expect(result["firstScans"]).toEqual(["legacy-a", "plain-a"]); + expect(result["secondScans"]).toEqual(["plain-b"]); + }, 30_000); +}); diff --git a/sdk/typescript/tests-ts/workbench-target-migration.test.ts b/sdk/typescript/tests-ts/workbench-target-migration.test.ts new file mode 100644 index 000000000..d1ad79897 --- /dev/null +++ b/sdk/typescript/tests-ts/workbench-target-migration.test.ts @@ -0,0 +1,389 @@ +import { spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "bun:test"; +import { PLUGIN_ROOT } from "./plugin-root.js"; + +const migrationProbe = String.raw` +import json +import sqlite3 +import sys +from pathlib import Path + +sys.path.insert(0, sys.argv[1]) + +from workbench_schema import MIGRATIONS, apply_migrations +from workbench_target_state import backfill_security_targets, stable_target_id + +scenario = sys.argv[2] +root = Path(sys.argv[3]) +timestamp = "2026-08-01T00:00:00Z" +backfill_calls = [] + +def backfill(connection): + backfill_calls.append(True) + backfill_security_targets(connection) + +connection = sqlite3.connect(":memory:") +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +apply_migrations(connection, MIGRATIONS, lambda: timestamp, backfill) +backfill_calls.clear() + +existing_path = str(root / "existing-repository") +missing_path = str(root / "deleted-repository") +connection.execute( + "INSERT INTO security_targets (id, current_path, display_name, created_at, updated_at) VALUES (?, ?, ?, ?, ?)", + ("target-existing", existing_path, "existing-repository", timestamp, timestamp), +) +connection.executemany( + "INSERT INTO workspaces (id, target_path, target_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?)", + [ + ("workspace-existing", existing_path, "target-existing", timestamp, timestamp), + ("workspace-empty", None, None, timestamp, timestamp), + ], +) + +def insert_scan(scan_id, workspace_id, target_path, target_id): + connection.execute( + "INSERT INTO scans (id, workspace_id, target_path, target_id, target_revision, scope, mode, scan_dir, status, phase, started_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + scan_id, + workspace_id, + target_path, + target_id, + "synthetic-revision", + ".", + "standard", + str(root / scan_id), + "complete", + "reporting", + timestamp, + timestamp, + timestamp, + ), + ) + +insert_scan("scan-existing", "workspace-existing", existing_path, "target-existing") + +if scenario == "orphan-scan": + insert_scan("scan-orphan", "workspace-existing", existing_path, None) + expected_target_id = "target-existing" + orphan_path = existing_path +elif scenario in ("dangling-scan", "dangling-workspace", "dangling-targets"): + connection.commit() + connection.execute("PRAGMA foreign_keys = OFF") + if scenario != "dangling-scan": + connection.execute( + "INSERT INTO workspaces (id, target_path, target_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?)", + ("workspace-orphan", missing_path, "target-dangling", timestamp, timestamp), + ) + if scenario != "dangling-workspace": + workspace = ( + "workspace-existing" if scenario == "dangling-scan" else "workspace-orphan" + ) + insert_scan("scan-orphan", workspace, missing_path, "target-dangling") + connection.commit() + connection.execute("PRAGMA foreign_keys = ON") + expected_target_id = stable_target_id(Path(missing_path)) + orphan_path = missing_path +else: + connection.execute( + "INSERT INTO workspaces (id, target_path, target_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?)", + ("workspace-orphan", missing_path, None, timestamp, timestamp), + ) + expected_target_id = stable_target_id(Path(missing_path)) + orphan_path = missing_path + if scenario == "orphan-workspace-and-scan": + insert_scan("scan-orphan", "workspace-orphan", missing_path, None) + +connection.commit() +violations_before_repair = len(connection.execute("PRAGMA foreign_key_check").fetchall()) +apply_migrations(connection, MIGRATIONS, lambda: timestamp, backfill) +apply_migrations(connection, MIGRATIONS, lambda: timestamp, backfill) + +def target_id(table, row_id): + row = connection.execute( + f"SELECT target_id FROM {table} WHERE id = ?", (row_id,) + ).fetchone() + return row["target_id"] if row is not None else None + +print(json.dumps({ + "backfillCalls": len(backfill_calls), + "emptyWorkspaceTargetId": target_id("workspaces", "workspace-empty"), + "existingScanTargetId": target_id("scans", "scan-existing"), + "existingWorkspaceTargetId": target_id("workspaces", "workspace-existing"), + "expectedTargetId": expected_target_id, + "foreignKeysEnforced": bool(connection.execute("PRAGMA foreign_keys").fetchone()[0]), + "foreignKeyViolationsBeforeRepair": violations_before_repair, + "foreignKeyViolationsAfterRepair": len( + connection.execute("PRAGMA foreign_key_check").fetchall() + ), + "migrationRecorded": connection.execute( + "SELECT name FROM schema_migrations WHERE version = 16" + ).fetchone()[0] == "stable repository targets", + "orphanPathExists": Path(orphan_path).exists(), + "orphanScanTargetId": target_id("scans", "scan-orphan"), + "orphanWorkspaceTargetId": target_id("workspaces", "workspace-orphan"), + "targetCount": connection.execute( + "SELECT count(*) FROM security_targets" + ).fetchone()[0], +})) +`; + +const repositoryIdentityMigrationProbe = String.raw` +import json +import sqlite3 +import sys + +sys.path.insert(0, sys.argv[1]) + +from workbench_schema import MIGRATIONS, apply_migrations +from workbench_target_state import backfill_security_targets + +scenario = sys.argv[2] +timestamp = "2026-08-01T00:00:00Z" +backfill_calls = [] + +def backfill(connection): + backfill_calls.append(True) + backfill_security_targets(connection) + +connection = sqlite3.connect(":memory:") +connection.row_factory = sqlite3.Row +historical_migration = next( + (migration for migration in MIGRATIONS if migration[0] == 29), None +) +historical = tuple( + migration for migration in MIGRATIONS if migration[0] < 30 and migration[0] != 29 +) +apply_migrations(connection, historical, lambda: timestamp, backfill) +backfill_calls.clear() +connection.execute( + "INSERT INTO security_targets (id, current_path, display_name, created_at, updated_at) VALUES (?, ?, ?, ?, ?)", + ("target-existing", "/synthetic/deleted-repository", "repository", timestamp, timestamp), +) + +if scenario == "recorded-without-column": + connection.execute( + "INSERT INTO schema_migrations VALUES (?, ?, ?)", + (30, "persist repository identities", timestamp), + ) +elif scenario == "recorded-without-index": + connection.execute( + "ALTER TABLE security_targets ADD COLUMN repository_identity TEXT" + ) + connection.execute( + "INSERT INTO schema_migrations VALUES (?, ?, ?)", + (30, "persist repository identities", timestamp), + ) + +connection.commit() +migrations = MIGRATIONS +if scenario == "out-of-order-historical-migration": + without_historical = tuple( + migration for migration in MIGRATIONS if migration[0] != 29 + ) + apply_migrations(connection, without_historical, lambda: timestamp, backfill) + migrations = tuple(sorted( + ( + *without_historical, + historical_migration or ( + 29, + "synthetic historical migration", + """ + CREATE TABLE historical_migration_fixture ( + id INTEGER PRIMARY KEY, + value TEXT + ); + + CREATE INDEX historical_migration_fixture_by_value + ON historical_migration_fixture(value); + """, + ), + ), + key=lambda migration: migration[0], + )) +apply_migrations(connection, migrations, lambda: timestamp, backfill) +apply_migrations(connection, migrations, lambda: timestamp, backfill) + +columns = { + row["name"]: row + for row in connection.execute("PRAGMA table_info(security_targets)") +} +indexes = { + row["name"]: row + for row in connection.execute("PRAGMA index_list(security_targets)") +} +print(json.dumps({ + "backfillCalls": len(backfill_calls), + "hasRepositoryIdentityColumn": "repository_identity" in columns, + "hasRepositoryIdentityIndex": "security_targets_by_repository_identity" in indexes, + "historicalMigrationApplied": connection.execute( + "SELECT 1 FROM schema_migrations WHERE version = 29" + ).fetchone() is not None, + "historicalMigrationInSource": historical_migration is not None, + "repositoryIdentityColumnIsNullable": not bool( + columns["repository_identity"]["notnull"] + ), + "repositoryIdentityIndexIsUnique": bool( + indexes["security_targets_by_repository_identity"]["unique"] + ), + "migrationName": connection.execute( + "SELECT name FROM schema_migrations WHERE version = 30" + ).fetchone()[0], + "targetId": connection.execute( + "SELECT id FROM security_targets" + ).fetchone()[0], +})) +`; + +describe("stable workbench target migration", () => { + test.each([ + ["orphan-scan", "reuses an existing target for an orphaned scan"], + ["orphan-workspace", "repairs a workspace without an orphaned scan"], + ["dangling-scan", "repairs a dangling scan foreign key independently"], + [ + "dangling-workspace", + "repairs a dangling workspace foreign key independently", + ], + [ + "dangling-targets", + "repairs dangling workspace and scan foreign keys atomically", + ], + [ + "orphan-workspace-and-scan", + "repairs a workspace and scan after their repository is deleted", + ], + ] as const)("%s: %s", (scenario) => { + const python = + Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py"); + expect(python).not.toBeNull(); + if (python === null) throw new Error("A Python interpreter is required."); + + const execution = spawnSync( + python, + [ + "-I", + "-B", + "-c", + migrationProbe, + join(PLUGIN_ROOT, "scripts"), + scenario, + join(tmpdir(), "codex-security-stable-target-migration", scenario), + ], + { encoding: "utf8", timeout: 10_000 }, + ); + + expect(execution.status, execution.stderr).toBe(0); + expect(execution.stderr).toBe(""); + + const result = JSON.parse(execution.stdout) as { + backfillCalls: number; + emptyWorkspaceTargetId: string | null; + existingScanTargetId: string; + existingWorkspaceTargetId: string; + expectedTargetId: string; + foreignKeysEnforced: boolean; + foreignKeyViolationsBeforeRepair: number; + foreignKeyViolationsAfterRepair: number; + migrationRecorded: boolean; + orphanPathExists: boolean; + orphanScanTargetId: string | null; + orphanWorkspaceTargetId: string | null; + targetCount: number; + }; + + expect(result).toMatchObject({ + backfillCalls: 1, + emptyWorkspaceTargetId: null, + existingScanTargetId: "target-existing", + existingWorkspaceTargetId: "target-existing", + foreignKeysEnforced: true, + foreignKeyViolationsBeforeRepair: + scenario === "dangling-targets" + ? 2 + : scenario.startsWith("dangling-") + ? 1 + : 0, + foreignKeyViolationsAfterRepair: 0, + migrationRecorded: true, + orphanPathExists: false, + targetCount: scenario === "orphan-scan" ? 1 : 2, + }); + expect(result.orphanScanTargetId).toBe( + scenario === "orphan-workspace" || scenario === "dangling-workspace" + ? null + : result.expectedTargetId, + ); + expect(result.orphanWorkspaceTargetId).toBe( + scenario === "orphan-scan" || scenario === "dangling-scan" + ? null + : result.expectedTargetId, + ); + }); + + test.each([ + [ + "unapplied", + "applies and backfills the new repository-identity migration", + ], + [ + "recorded-without-column", + "repairs a recorded migration missing its column and index", + ], + [ + "recorded-without-index", + "repairs a recorded migration missing only its index", + ], + [ + "out-of-order-historical-migration", + "applies an unavailable migration after a newer migration is recorded", + ], + ] as const)("%s: %s", (scenario) => { + const python = + Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py"); + expect(python).not.toBeNull(); + if (python === null) throw new Error("A Python interpreter is required."); + + const execution = spawnSync( + python, + [ + "-I", + "-B", + "-c", + repositoryIdentityMigrationProbe, + join(PLUGIN_ROOT, "scripts"), + scenario, + ], + { encoding: "utf8", timeout: 10_000 }, + ); + + expect(execution.status, execution.stderr).toBe(0); + expect(execution.stderr).toBe(""); + const result = JSON.parse(execution.stdout) as { + backfillCalls: number; + hasRepositoryIdentityColumn: boolean; + hasRepositoryIdentityIndex: boolean; + historicalMigrationApplied: boolean; + historicalMigrationInSource: boolean; + migrationName: string; + repositoryIdentityColumnIsNullable: boolean; + repositoryIdentityIndexIsUnique: boolean; + targetId: string; + }; + expect(result).toEqual({ + backfillCalls: 1, + hasRepositoryIdentityColumn: true, + hasRepositoryIdentityIndex: true, + historicalMigrationApplied: + scenario === "out-of-order-historical-migration" || + result.historicalMigrationInSource, + historicalMigrationInSource: result.historicalMigrationInSource, + migrationName: "persist repository identities", + repositoryIdentityColumnIsNullable: true, + repositoryIdentityIndexIsUnique: false, + targetId: "target-existing", + }); + }); +}); From 20b170e3cab5ee54378dc335fd132c3f93bb5cea Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 15 Aug 2026 01:43:33 -0700 Subject: [PATCH 02/21] fix(workbench): verify repository generation across platforms --- .../scripts/workbench_target_state.py | 12 ++++- sdk/typescript/tests-ts/cli-workbench.test.ts | 4 +- .../repository-feedback-aliases.test.ts | 8 +++- .../tests-ts/repository-findings.test.ts | 8 +++- .../workbench-repository-identity.test.ts | 48 +++++++++++++++++-- 5 files changed, 73 insertions(+), 7 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index e15813c8a..9f74c737b 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -6,6 +6,7 @@ import hashlib import os import sqlite3 +import stat import sys from datetime import datetime, timezone from pathlib import Path @@ -68,11 +69,20 @@ def repository_identity(target: Path | str) -> str | None: canonical_directory = os.path.normcase(os.path.realpath(common_directory)) try: metadata = Path(canonical_directory).stat() + generation = (Path(canonical_directory) / "description").lstat() except OSError: return None + if not stat.S_ISREG(generation.st_mode): + return None device = serialize_filesystem_identity(metadata.st_dev) inode = serialize_filesystem_identity(metadata.st_ino) - material = f"git-common-dir\0{canonical_directory}\0{device}\0{inode}\0{relative}" + generation_device = serialize_filesystem_identity(generation.st_dev) + generation_inode = serialize_filesystem_identity(generation.st_ino) + material = ( + f"git-common-dir\0{canonical_directory}\0{device}\0{inode}\0" + f"git-description\0{generation_device}\0{generation_inode}\0" + f"{generation.st_ctime_ns}\0{relative}" + ) return f"repository_sha256_{hashlib.sha256(material.encode()).hexdigest()}" diff --git a/sdk/typescript/tests-ts/cli-workbench.test.ts b/sdk/typescript/tests-ts/cli-workbench.test.ts index 620709ec3..51f5a3106 100644 --- a/sdk/typescript/tests-ts/cli-workbench.test.ts +++ b/sdk/typescript/tests-ts/cli-workbench.test.ts @@ -98,7 +98,9 @@ describe("CLI workbench", () => { }, ])( "combines open findings from the $name", - async ({ requested, related }) => { + async ({ requested: requestedPath, related: relatedPath }) => { + const requested = resolve(requestedPath); + const related = resolve(relatedPath); const calls: Array = []; const findingsByTarget: Record = { requested: [ diff --git a/sdk/typescript/tests-ts/repository-feedback-aliases.test.ts b/sdk/typescript/tests-ts/repository-feedback-aliases.test.ts index b5f2e3a19..35709a5a3 100644 --- a/sdk/typescript/tests-ts/repository-feedback-aliases.test.ts +++ b/sdk/typescript/tests-ts/repository-feedback-aliases.test.ts @@ -8,6 +8,7 @@ test("reuses reviewed feedback only across matching persisted repository identit const probe = ` import json, os, sqlite3, sys, tempfile sys.path.insert(0, sys.argv[1]) +from filesystem_identity import serialize_filesystem_identity from workbench_feedback import get_scan_feedback scenario = sys.argv[2] @@ -125,7 +126,12 @@ if scenario == "identities": ) connection.execute( "UPDATE scans SET target_path = ?, target_device = ?, target_inode = ? WHERE target_id = ?", - (reused_path, metadata.st_dev, metadata.st_ino + 1, "primary"), + ( + reused_path, + serialize_filesystem_identity(metadata.st_dev), + serialize_filesystem_identity(metadata.st_ino + 1), + "primary", + ), ) result["reusedPath"] = feedback("primary") result["deletedAlias"] = feedback("linked") diff --git a/sdk/typescript/tests-ts/repository-findings.test.ts b/sdk/typescript/tests-ts/repository-findings.test.ts index 7a49c8171..e8b3af6b0 100644 --- a/sdk/typescript/tests-ts/repository-findings.test.ts +++ b/sdk/typescript/tests-ts/repository-findings.test.ts @@ -120,6 +120,7 @@ test("shares findings only between explicitly identified repository and scope al const probe = ` import argparse, json, os, sqlite3, sys, tempfile sys.path.insert(0, sys.argv[1]) +from filesystem_identity import serialize_filesystem_identity import workbench_native_indexes as indexes connection = sqlite3.connect(":memory:") @@ -234,7 +235,12 @@ with tempfile.TemporaryDirectory() as reused_path: ) connection.execute( "UPDATE scans SET target_path = ?, target_device = ?, target_inode = ? WHERE target_id = ?", - (reused_path, metadata.st_dev, metadata.st_ino + 1, "primary"), + ( + reused_path, + serialize_filesystem_identity(metadata.st_dev), + serialize_filesystem_identity(metadata.st_ino + 1), + "primary", + ), ) result["reusedPath"] = findings("primary") result["deletedAlias"] = findings("linked") diff --git a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts index f53e141e1..e032a7f5a 100644 --- a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts +++ b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts @@ -198,20 +198,53 @@ if scenario == "identity": metadata = Path(common).stat() device = serialize_filesystem_identity(metadata.st_dev) inode = serialize_filesystem_identity(metadata.st_ino) + marker = Path(common) / "description" + generation = marker.lstat() + generation_device = serialize_filesystem_identity(generation.st_dev) + generation_inode = serialize_filesystem_identity(generation.st_ino) expected = "repository_sha256_" + hashlib.sha256( - f"git-common-dir\0{common}\0{device}\0{inode}\0.".encode() + ( + f"git-common-dir\0{common}\0{device}\0{inode}\0" + f"git-description\0{generation_device}\0{generation_inode}\0" + f"{generation.st_ctime_ns}\0." + ).encode() ).hexdigest() forged_identity = repository_identity(forged_worktree()) + (repository / "service-a" / "service.py").write_text("value = 2\n") + git(repository, "add", ".") + git(repository, "commit", "-qm", "ordinary update") + commit_independent = repository_identity(repository) == identities["repository"] + additional_worktree = root / "additional-worktree" + git(repository, "worktree", "add", "--detach", "-q", str(additional_worktree), "HEAD") + worktree_independent = repository_identity(repository) == identities["repository"] + git(repository, "worktree", "remove", "--force", str(additional_worktree)) git( repository, "remote", "set-url", "origin", "https://different-user:DIFFERENT_SYNTHETIC_PASSWORD@example.test/other/repo.git", ) + remote_independent = repository_identity(repository) == identities["repository"] + saved_marker = root / "saved-description" + marker.rename(saved_marker) + missing_marker_identity = repository_identity(repository) + try: + marker.symlink_to(saved_marker) + except (NotImplementedError, OSError): + symlink_supported = False + symlink_marker_identity = None + else: + symlink_supported = True + symlink_marker_identity = repository_identity(repository) print(json.dumps({ "identities": identities, "expected": expected, "forgedIdentity": forged_identity, - "remoteIndependent": repository_identity(repository) == identities["repository"], + "commitIndependent": commit_independent, + "worktreeIndependent": worktree_independent, + "remoteIndependent": remote_independent, + "missingMarkerIdentity": missing_marker_identity, + "symlinkSupported": symlink_supported, + "symlinkMarkerIdentity": symlink_marker_identity, "targetIdsPreserved": all( target_ids[name] == stable_target_id(target) for name, target in targets.items() @@ -421,7 +454,12 @@ elif scenario == "git-replacement": add_scan("original-alias", worktree) original_identity = repository_identity(repository) original_metadata = repository.stat() - shutil.rmtree(repository / ".git") + + def remove_read_only(operation, path, _error): + os.chmod(path, 0o700) + operation(path) + + shutil.rmtree(repository / ".git", onerror=remove_read_only) git(repository, "init", "-q") git(repository, "add", ".") git(repository, "commit", "-qm", "replacement") @@ -532,7 +570,11 @@ describe("durable workbench repository identities", () => { expect(identities["serviceA"]).not.toBe(identities["serviceB"]); expect(identities["clone"]).not.toBe(identities["repository"]); expect(result["forgedIdentity"]).toBeNull(); + expect(result["commitIndependent"]).toBe(true); + expect(result["worktreeIndependent"]).toBe(true); expect(result["remoteIndependent"]).toBe(true); + expect(result["missingMarkerIdentity"]).toBeNull(); + expect(result["symlinkMarkerIdentity"]).toBeNull(); expect(result["targetIdsPreserved"]).toBe(true); expect(JSON.stringify(result["stored"])).not.toContain( "SYNTHETIC_PASSWORD", From 4f1eb91dcbe063452b2b0b5791fe15e00a5b3cfc Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 15 Aug 2026 12:44:29 -0700 Subject: [PATCH 03/21] fix(workbench): harden and simplify worktree history --- .../_bundled_plugin/scripts/workbench_cli.py | 1 - .../scripts/workbench_native_indexes.py | 20 +- .../scripts/workbench_scan_history.py | 38 ++-- .../scripts/workbench_target_state.py | 54 ++++-- sdk/typescript/src/cli.ts | 69 ++----- sdk/typescript/tests-ts/cli-workbench.test.ts | 182 +++--------------- .../tests-ts/repository-findings.test.ts | 35 ++++ .../workbench-canonical-paths.test.ts | 67 +++++++ .../workbench-repository-identity.test.ts | 173 +++++++++-------- 9 files changed, 311 insertions(+), 328 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py b/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py index 15cc36115..56f8997c7 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py @@ -131,7 +131,6 @@ def parse_args(description: str) -> argparse.Namespace: list_scans.add_argument("--status", choices=("running", "complete", "failed", "canceled")) list_scans.add_argument("--mode", choices=MODES) list_scans.add_argument("--repository") - list_scans.add_argument("--worktrees-only", action="store_true", help=argparse.SUPPRESS) list_scans.add_argument("--scan-root") list_scans.add_argument("--offset", type=non_negative_int, default=0) list_scans.add_argument("--limit", type=positive_int) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py index f341af1a3..9d1df1032 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py @@ -307,16 +307,28 @@ def list_repositories( ): latest_scan_by_target.setdefault(row["target_id"], scans_by_id[row["id"]]) - open_findings_by_target = Counter( - row["target_id"] for row in _indexed_findings(connection) if row["status"] == "open" - ) targets = {row["id"]: row for row in connection.execute("SELECT * FROM security_targets")} + + def repository_group(target_id: str) -> tuple[str, str]: + target = targets[target_id] + identity = ( + target["repository_identity"] if "repository_identity" in target.keys() else None + ) + return ("target", target_id) if identity is None else ("repository", identity) + + open_findings_by_repository = Counter( + repository_group(row["target_id"]) + for row in _indexed_findings(connection) + if row["status"] == "open" + ) repositories = [ { "checkoutAvailable": Path(target["current_path"]).is_dir(), "displayName": target["display_name"], "latestScan": latest_scan, - "openFindingsCount": open_findings_by_target.get(target_id, 0), + "openFindingsCount": open_findings_by_repository.get( + repository_group(target_id), 0 + ), "scanCount": scan_count_by_target[target_id], "targetId": target_id, "targetPath": target["current_path"], diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py index ece0e2cfb..37b93f17e 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py @@ -29,14 +29,13 @@ def _same_repository( after: sqlite3.Row, *, after_identity: tuple[str | None, tuple[str, str] | None, str | None] | None = None, - worktrees_only: bool = False, ) -> bool: before_target_id = before["target_id"] if before_target_id and before_target_id == after["target_id"]: return True before_target = Path(before["target_path"]) after_target = Path(after["target_path"]) - if before_target.resolve() == after_target.resolve(): + if str(before_target.resolve()) == str(after_target.resolve()): return True before_stored_identity = ( before["repository_identity"] if "repository_identity" in before.keys() else None @@ -60,14 +59,13 @@ def _same_repository( ) if before_live_identity is not None and before_live_identity == after_live_identity: return True - if worktrees_only or before_live_identity is None or after_live_identity is None: + if before_live_identity is None or after_live_identity is None: return False - before_origin = _repository_origin(before_target) after_origin = ( _repository_origin(after_target) if after_identity is None else after_identity[1] ) - if before_origin is None or before_origin != after_origin: + if after_origin is None or _repository_origin(before_target) != after_origin: return False before_relative = repository_relative_path(before_target) after_relative = ( @@ -110,7 +108,6 @@ def list_scans( values: list[Any] = [] if args is not None and args.repository: repository = Path(args.repository).expanduser().resolve() - worktrees_only = bool(getattr(args, "worktrees_only", False)) target_columns = { row["name"] for row in connection.execute("PRAGMA table_info(security_targets)") } @@ -161,8 +158,8 @@ def list_scans( ) else live_identity ), - None if worktrees_only else _repository_origin(repository), - None if worktrees_only else repository_relative_path(repository), + None, + None, ) related_targets = ( [ @@ -177,7 +174,6 @@ def list_scans( target, requested_repository, after_identity=requested_identity, - worktrees_only=worktrees_only, ) ] if ownership_matches @@ -188,21 +184,15 @@ def list_scans( if identity_matches else ["0"] ) - if worktrees_only: - for target in related_targets: - target_path = Path(target["target_path"]) - repository_clauses.append( - _owned_scan_clause( - target_path, - check_ownership, - values, - target_id=target["target_id"], - ) + for target in related_targets: + repository_clauses.append( + _owned_scan_clause( + Path(target["target_path"]), + check_ownership, + values, + target_id=target["target_id"], ) - elif related_targets: - placeholders = ", ".join("?" for _ in related_targets) - repository_clauses.append(f"scans.target_id IN ({placeholders})") - values.extend(target["target_id"] for target in related_targets) + ) clauses.append(f"({' OR '.join(repository_clauses)})") if args is not None and args.scan_root: scan_root = str(Path(args.scan_root).expanduser().resolve()) @@ -440,7 +430,7 @@ def list_unmatched_scan_pairs( selected = [ scan for scan in connection.execute(scan_query) - if Path(scan["target_path"]).resolve() == repository + if str(Path(scan["target_path"]).resolve()) == str(repository) or _same_repository( scan, requested, diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index 9f74c737b..538958088 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -6,7 +6,7 @@ import hashlib import os import sqlite3 -import stat +import subprocess import sys from datetime import datetime, timezone from pathlib import Path @@ -40,7 +40,41 @@ def _repository_worktree(target: Path) -> tuple[Path, str] | None: relative = Path(os.path.realpath(target)).relative_to(canonical_root) except (OSError, ValueError): return None - return canonical_root, os.path.normcase(os.fspath(relative)).replace(os.sep, "/") + return canonical_root, relative.as_posix() + + +def _repository_birth_time_ns(path: str, metadata: os.stat_result) -> int | None: + birth_time_ns = getattr(metadata, "st_birthtime_ns", None) + if birth_time_ns is not None: + return birth_time_ns if birth_time_ns > 0 else None + if os.name == "nt": + return metadata.st_ctime_ns if metadata.st_ctime_ns > 0 else None + birth_time = getattr(metadata, "st_birthtime", None) + if birth_time is not None: + return int(birth_time * 1_000_000_000) if birth_time > 0 else None + if not sys.platform.startswith("linux"): + return None + try: + result = subprocess.run( + ["stat", "--format=%.9W", "--", path], + check=False, + capture_output=True, + text=True, + env={**os.environ, "LC_ALL": "C"}, + ) + except OSError: + return None + seconds, separator, nanoseconds = result.stdout.strip().partition(".") + if ( + result.returncode != 0 + or separator != "." + or not seconds.isdecimal() + or len(nanoseconds) != 9 + or not nanoseconds.isdecimal() + ): + return None + birth_time_ns = int(seconds) * 1_000_000_000 + int(nanoseconds) + return birth_time_ns if birth_time_ns > 0 else None def repository_identity(target: Path | str) -> str | None: @@ -58,30 +92,26 @@ def repository_identity(target: Path | str) -> str | None: registered = git_output(target, "worktree", "list", "--porcelain", "-z") if registered is None: return None - canonical_root = os.path.normcase(os.fspath(worktree_root)) + canonical_root = os.fspath(worktree_root) if not any( - os.path.normcase(os.path.realpath(record.removeprefix("worktree "))) - == canonical_root + os.path.realpath(record.removeprefix("worktree ")) == canonical_root for record in registered.split("\0") if record.startswith("worktree ") ): return None - canonical_directory = os.path.normcase(os.path.realpath(common_directory)) + canonical_directory = os.path.realpath(common_directory) try: metadata = Path(canonical_directory).stat() - generation = (Path(canonical_directory) / "description").lstat() except OSError: return None - if not stat.S_ISREG(generation.st_mode): + birth_time_ns = _repository_birth_time_ns(canonical_directory, metadata) + if birth_time_ns is None: return None device = serialize_filesystem_identity(metadata.st_dev) inode = serialize_filesystem_identity(metadata.st_ino) - generation_device = serialize_filesystem_identity(generation.st_dev) - generation_inode = serialize_filesystem_identity(generation.st_ino) material = ( f"git-common-dir\0{canonical_directory}\0{device}\0{inode}\0" - f"git-description\0{generation_device}\0{generation_inode}\0" - f"{generation.st_ctime_ns}\0{relative}" + f"{birth_time_ns}\0{relative}" ) return f"repository_sha256_{hashlib.sha256(material.encode()).hexdigest()}" diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index f751d9130..917b6df51 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -919,63 +919,20 @@ export async function main( ); return presentHistory( await history( - ["list-scans", "--repository", repository, "--worktrees-only"], + ["list-scans", "--repository", repository], async (value): Promise => { - const targetIds = new Set( - (value["scans"] as JsonObject[]) - .map((scan) => scan["targetId"]) - .filter( - (targetId): targetId is string => - typeof targetId === "string", - ), - ); - const occurrences = new Set(); - const findings: JsonObject[] = []; - for (const targetId of targetIds) { - const targetFindings = await listRepositoryFindings( - dependencies.runWorkbench, - targetId, - ); - for (const finding of targetFindings ?? []) { - const occurrenceId = finding["occurrenceId"]; - if (typeof occurrenceId === "string") { - if (occurrences.has(occurrenceId)) continue; - occurrences.add(occurrenceId); - } - findings.push(finding); - } - } - findings.sort((before, after) => { - const rank = (finding: JsonObject): number => { - const severity = ( - finding["severity"] as JsonObject | undefined - )?.["level"]; - const index = DISPLAY_SEVERITIES.indexOf( - severity as SeverityLevel, - ); - return index === -1 ? DISPLAY_SEVERITIES.length : index; - }; - const severity = rank(before) - rank(after); - if (severity !== 0) return severity; - - const beforeCreated = before["createdAt"]; - const afterCreated = after["createdAt"]; - if ( - typeof beforeCreated === "string" && - typeof afterCreated === "string" && - beforeCreated !== afterCreated - ) { - return beforeCreated < afterCreated ? 1 : -1; - } - - const beforeId = before["occurrenceId"]; - const afterId = after["occurrenceId"]; - if (typeof beforeId !== "string" || typeof afterId !== "string") { - return 0; - } - return beforeId < afterId ? -1 : beforeId > afterId ? 1 : 0; - }); - return { repository, findings }; + const scans = value["scans"] as JsonObject[]; + const target = + scans.find((scan) => scan["targetPath"] === repository) ?? + scans[0]; + const findings = + target === undefined + ? [] + : await listRepositoryFindings( + dependencies.runWorkbench, + target["targetId"] as string, + ); + return { repository, findings: findings ?? [] }; }, ), "findings", diff --git a/sdk/typescript/tests-ts/cli-workbench.test.ts b/sdk/typescript/tests-ts/cli-workbench.test.ts index 51f5a3106..2e2ea9532 100644 --- a/sdk/typescript/tests-ts/cli-workbench.test.ts +++ b/sdk/typescript/tests-ts/cli-workbench.test.ts @@ -18,12 +18,7 @@ describe("CLI workbench", () => { const stdout = capture(); const calls: Array = []; const responses: JsonObject[] = [ - { - scans: [ - { targetId: "selected", targetPath: repository }, - { targetId: "selected", targetPath: repository }, - ], - }, + { scans: [{ targetId: "selected", targetPath: repository }] }, { findings: [{ title: "Finding 1" }], nextOffset: 1 }, { findings: [{ title: "Finding 2" }], nextOffset: null }, ]; @@ -37,12 +32,7 @@ describe("CLI workbench", () => { }), ), ).toBe(0); - expect(calls[0]).toEqual([ - "list-scans", - "--repository", - repository, - "--worktrees-only", - ]); + expect(calls[0]).toEqual(["list-scans", "--repository", repository]); expect(calls[1]).toEqual([ "list-global-findings", "--target-id", @@ -81,171 +71,55 @@ describe("CLI workbench", () => { }); test.each([ - { - name: "canonical checkout and linked worktree", - requested: "/repositories/project", - related: "/repositories/project-worktree", - }, - { - name: "linked worktree and canonical checkout", - requested: "/repositories/project-worktree", - related: "/repositories/project", - }, - { - name: "explicitly linked repository alias", - requested: "/repositories/project", - related: "/repositories/linked-alias", - }, + { requestedScan: false, selectedTarget: "linked" }, + { requestedScan: true, selectedTarget: "requested" }, ])( - "combines open findings from the $name", - async ({ requested: requestedPath, related: relatedPath }) => { - const requested = resolve(requestedPath); - const related = resolve(relatedPath); + "lists findings through one trusted target when requested scan exists: $requestedScan", + async ({ requestedScan, selectedTarget }) => { + const repository = resolve("/current/repository"); + const linked = resolve("/current/repository-worktree"); + const findings: JsonObject[] = [ + { occurrenceId: "requested-finding", status: "open" }, + { occurrenceId: "linked-finding", status: "open" }, + ]; const calls: Array = []; - const findingsByTarget: Record = { - requested: [ - { - occurrenceId: "requested-medium", - title: "Requested checkout finding", - severity: { level: "medium" }, - createdAt: "2026-08-13T12:00:00Z", - status: "open", - }, - { - occurrenceId: "duplicate-high", - title: "Duplicated finding", - severity: { level: "high" }, - createdAt: "2026-08-12T12:00:00Z", - status: "open", - }, - ], - related: [ - { - occurrenceId: "related-high", - title: "Related checkout finding", - severity: { level: "high" }, - createdAt: "2026-08-14T12:00:00Z", - status: "open", - }, - { - occurrenceId: "duplicate-high", - title: "Duplicated finding", - severity: { level: "high" }, - createdAt: "2026-08-12T12:00:00Z", - status: "open", - }, - { - occurrenceId: "closed-critical", - title: "Closed finding", - severity: { level: "critical" }, - createdAt: "2026-08-15T12:00:00Z", - status: "closed", - }, - ], - }; const stdout = capture(); expect( await main( - ["findings", "list", requested, "--json"], + ["findings", "list", "--json"], stdout.stream, capture().stream, dependencies({ onWorkbench: (args): JsonObject => { calls.push(args); - if (args[0] === "list-scans") { - return { - scans: [ - { targetId: "requested", targetPath: requested }, - { targetId: "related", targetPath: related }, - { targetId: "requested", targetPath: requested }, - ], - }; - } - const targetId = args[args.indexOf("--target-id") + 1]!; - const requestedStatus = args[args.indexOf("--status") + 1]; - return { - findings: findingsByTarget[targetId]!.filter( - (finding) => finding["status"] === requestedStatus, - ), - nextOffset: null, - }; + return args[0] === "list-scans" + ? { + scans: [ + { targetId: "linked", targetPath: linked }, + ...(requestedScan + ? [{ targetId: "requested", targetPath: repository }] + : []), + ], + } + : { findings, nextOffset: null }; }, }), ), ).toBe(0); expect(calls).toEqual([ - ["list-scans", "--repository", requested, "--worktrees-only"], + ["list-scans", "--repository", repository], [ "list-global-findings", "--target-id", - "requested", + selectedTarget, "--status", "open", ], - ["list-global-findings", "--target-id", "related", "--status", "open"], ]); - expect(JSON.parse(stdout.text())).toEqual({ - repository: requested, - findings: [ - findingsByTarget["related"]![0], - findingsByTarget["requested"]![1], - findingsByTarget["requested"]![0], - ], - }); + expect(JSON.parse(stdout.text())).toEqual({ repository, findings }); }, ); - test("does not query findings for same-origin-only clones", async () => { - const repository = resolve("/current/repository"); - const calls: Array = []; - const stdout = capture(); - const safeFinding: JsonObject = { - occurrenceId: "requested-finding", - title: "Finding in the requested worktree", - status: "open", - }; - expect( - await main( - ["findings", "list", "--json"], - stdout.stream, - capture().stream, - dependencies({ - onWorkbench: (args): JsonObject => { - calls.push(args); - if (args[0] === "list-scans") { - const scans: JsonObject[] = [ - { targetId: "requested", targetPath: repository }, - ]; - if (!args.includes("--worktrees-only")) { - scans.push({ - targetId: "same-origin-clone", - targetPath: `${repository}-clone`, - }); - } - return { scans }; - } - const targetId = args[args.indexOf("--target-id") + 1]; - return { - findings: - targetId === "requested" - ? [safeFinding] - : [{ occurrenceId: "private-clone-finding", status: "open" }], - nextOffset: null, - }; - }, - }), - ), - ).toBe(0); - expect(calls).toEqual([ - ["list-scans", "--repository", repository, "--worktrees-only"], - ["list-global-findings", "--target-id", "requested", "--status", "open"], - ]); - expect(JSON.parse(stdout.text())).toEqual({ - repository, - findings: [safeFinding], - }); - }); - test("returns no findings when no related repository has been scanned", async () => { const calls: Array = []; const stdout = capture(); @@ -263,9 +137,7 @@ describe("CLI workbench", () => { }), ), ).toBe(0); - expect(calls).toEqual([ - ["list-scans", "--repository", repository, "--worktrees-only"], - ]); + expect(calls).toEqual([["list-scans", "--repository", repository]]); expect(JSON.parse(stdout.text())).toEqual({ repository, findings: [] }); }); diff --git a/sdk/typescript/tests-ts/repository-findings.test.ts b/sdk/typescript/tests-ts/repository-findings.test.ts index e8b3af6b0..9a82060a0 100644 --- a/sdk/typescript/tests-ts/repository-findings.test.ts +++ b/sdk/typescript/tests-ts/repository-findings.test.ts @@ -137,6 +137,7 @@ origin = "https://example.invalid/synthetic/repository" connection.executemany("INSERT INTO security_targets VALUES (?, ?, ?, ?, ?)", [ ("primary", "/primary", "Primary", "common-git-directory::.", origin), ("linked", "/linked", "Linked", "common-git-directory::.", origin), + ("empty-alias", "/empty-alias", "Empty alias", "common-git-directory::.", origin), ("same-origin-clone", "/clone", "Clone", "independent-git-directory::.", origin), ("different-scope", "/primary/packages/api", "Scoped", "common-git-directory::packages/api", origin), ("unknown-first", "/unknown-first", "Unknown first", None, origin), @@ -154,6 +155,7 @@ def add_finding(occurrence, finding, scan, severity="high"): for scan_id, target, day in [ ("primary-old", "primary", 1), + ("empty-alias-scan", "empty-alias", 1), ("linked-reviewed", "linked", 2), ("primary-open", "primary", 3), ("linked-latest", "linked", 4), @@ -200,6 +202,15 @@ def findings(target, status="open", limit=20, offset=0): arguments = argparse.Namespace(limit=limit, offset=offset, query=None, severity=None, status=status, target_id=target) return indexes.list_global_findings(connection, arguments) +indexes.scan_history.list_scans = lambda database: { + "scans": [ + {"scanId": scan["id"], "targetId": scan["target_id"]} + for scan in database.execute("SELECT id, target_id FROM scans") + ] +} +repository_arguments = argparse.Namespace( + query=None, target_id=None, status="open_findings", limit=None, offset=0 +) result = { "primary": findings("primary"), "linked": findings("linked"), @@ -212,6 +223,8 @@ result = { "firstPage": findings("primary", limit=2), "secondPage": findings("primary", limit=2, offset=2), "thirdPage": findings("primary", limit=2, offset=4), + "repositories": indexes.list_repositories(connection), + "openRepositories": indexes.list_repositories(connection, repository_arguments), } connection.execute( "INSERT INTO finding_triage VALUES (?, ?, ?, ?)", @@ -259,6 +272,7 @@ print(json.dumps(result)) { findings: Array>; nextOffset: number | null; + repositories?: Array>; } >; const primary = result["primary"]!.findings; @@ -321,6 +335,27 @@ print(json.dumps(result)) expect(result["unknownSecond"]!.findings).toMatchObject([ { findingId: "unknown-second-only", targetId: "unknown-second" }, ]); + expect( + Object.fromEntries( + result["repositories"]!.repositories!.map((repository) => [ + repository["targetId"], + repository["openFindingsCount"], + ]), + ), + ).toEqual({ + primary: 4, + linked: 4, + "empty-alias": 4, + "same-origin-clone": 1, + "different-scope": 1, + "unknown-first": 1, + "unknown-second": 1, + }); + expect( + result["openRepositories"]!.repositories!.map( + (repository) => repository["targetId"], + ), + ).toContain("empty-alias"); expect(result["firstPage"]!.nextOffset).toBe(2); expect(result["secondPage"]!.nextOffset).toBeNull(); expect(result["thirdPage"]!.nextOffset).toBeNull(); diff --git a/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts b/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts index dd04b8236..0d64d4bf4 100644 --- a/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts +++ b/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts @@ -52,6 +52,60 @@ const simulatedPathProbe = [ "print(json.dumps({'accepted': accepted, 'nativePathEquality': path_type(supplied) == path_type(resolved), 'resolvedPath': result}))", ].join("\n"); +const caseSensitiveWindowsHistoryProbe = [ + "import json, sqlite3, sys", + "from pathlib import PureWindowsPath", + "from types import SimpleNamespace", + "sys.path.insert(0, sys.argv[1])", + "import workbench_scan_history as history", + "class WindowsPath(PureWindowsPath):", + " def expanduser(self):", + " return self", + " def resolve(self, strict=False):", + " return self", + "history.Path = WindowsPath", + "connection = sqlite3.connect(':memory:')", + "connection.row_factory = sqlite3.Row", + "connection.executescript('''", + "CREATE TABLE security_targets (id TEXT, current_path TEXT, repository_identity TEXT);", + "CREATE TABLE scans (id TEXT, target_id TEXT, target_path TEXT, status TEXT, started_at TEXT);", + "CREATE TABLE scan_comparisons (before_scan_id TEXT, after_scan_id TEXT);", + "''')", + "def target(identifier, path, identity):", + " return connection.execute('SELECT ? AS target_id, ? AS target_path, ? AS repository_identity', (identifier, str(WindowsPath(path)), identity)).fetchone()", + "upper_root = target('upper-root', 'D:/Repository', 'root-upper')", + "lower_root = target('lower-root', 'D:/repository', 'root-lower')", + "upper_scope = target('upper-scope', 'D:/Repository/Service', 'scope-upper')", + "lower_scope = target('lower-scope', 'D:/Repository/service', 'scope-lower')", + "linked_scope = target('linked-scope', 'E:/Linked/Service', 'scope-upper')", + "clone_scope = target('clone-scope', 'D:/Clone/Service', 'scope-clone')", + "same_path = target('same-path', 'D:/Repository/Service', None)", + "targets = [upper_root, lower_root, upper_scope, lower_scope, linked_scope, clone_scope]", + "identities = {entry['target_path']: entry['repository_identity'] for entry in targets}", + "history.repository_identity = lambda path: identities.get(str(path))", + "history.repository_relative_path = lambda path: WindowsPath(path).name", + "history._repository_origin = lambda path: ('example.test', 'synthetic/repository')", + "root_group = ('root-upper', None, None)", + "scope_group = ('scope-upper', None, None)", + "checks = {", + " 'nativeWindowsScopeEquality': WindowsPath(upper_scope['target_path']) == WindowsPath(lower_scope['target_path']),", + " 'nativeWindowsRootEquality': WindowsPath(upper_root['target_path']) == WindowsPath(lower_root['target_path']),", + " 'caseSensitiveScopesMatch': history._same_repository(lower_scope, upper_scope, after_identity=scope_group),", + " 'caseSensitiveRootsMatch': history._same_repository(lower_root, upper_root, after_identity=root_group),", + " 'linkedWorktreeMatches': history._same_repository(linked_scope, upper_scope, after_identity=scope_group),", + " 'sameOriginCloneMatches': history._same_repository(clone_scope, upper_scope, after_identity=scope_group),", + " 'exactResolvedPathMatches': history._same_repository(same_path, upper_scope, after_identity=scope_group),", + "}", + "for entry in (upper_scope, lower_scope):", + " connection.execute('INSERT INTO security_targets VALUES (?, ?, ?)', (entry['target_id'], entry['target_path'], entry['repository_identity']))", + "connection.execute('INSERT INTO scans VALUES (?, ?, ?, ?, ?)', ('lower-scan', lower_scope['target_id'], lower_scope['target_path'], 'complete', '2026-08-15T00:00:00Z'))", + "history._require_current_target_owner = lambda *args: None", + "history._same_repository = lambda *args, **kwargs: False", + "selected = history.list_unmatched_scan_pairs(connection, SimpleNamespace(repository=upper_scope['target_path'], force=False), backfill_finding_details=lambda *args: None, read_coverage=lambda scan: {})", + "checks['caseSensitiveUnmatchedScanCount'] = selected['scanCount']", + "print(json.dumps(checks))", +].join("\n"); + const realFilesystemProbe = [ "import json, sys", "from pathlib import Path", @@ -178,6 +232,19 @@ describe("bundled workbench canonical paths", () => { }); }); + test("keeps case-sensitive Windows repository roots and scopes distinct", () => { + expect(runPythonProbe(caseSensitiveWindowsHistoryProbe)).toEqual({ + nativeWindowsScopeEquality: true, + nativeWindowsRootEquality: true, + caseSensitiveScopesMatch: false, + caseSensitiveRootsMatch: false, + linkedWorktreeMatches: true, + sameOriginCloneMatches: false, + exactResolvedPathMatches: true, + caseSensitiveUnmatchedScanCount: 0, + }); + }); + testCaseSensitive( "rejects case-differing symlinks at every workbench and finalizer boundary", async () => { diff --git a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts index e032a7f5a..5a7b382bf 100644 --- a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts +++ b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts @@ -50,7 +50,7 @@ function fixture(): { const clone = join(root, "same-origin-clone"); mkdirSync(repository); git(repository, "init", "-q"); - for (const service of ["service-a", "service-b"]) { + for (const service of ["service-a", "service-b", "MixedCase"]) { mkdirSync(join(repository, service)); writeFileSync(join(repository, service, "service.py"), "value = 1\n"); } @@ -67,7 +67,6 @@ function fixture(): { const probe = String.raw` import argparse -import hashlib import json import os import shutil @@ -75,6 +74,8 @@ import sqlite3 import subprocess import sys from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch sys.path.insert(0, sys.argv[1]) @@ -82,10 +83,12 @@ import workbench_scan_history as history from filesystem_identity import serialize_filesystem_identity from workbench_schema import MIGRATIONS, apply_migrations from workbench_target_state import ( + _repository_birth_time_ns, backfill_repository_identities, backfill_security_targets, ensure_security_target, repository_identity, + repository_relative_path, stable_target_id, ) @@ -152,10 +155,9 @@ def add_scan(scan_id, target, ownership="current", verify_ownership=True): return target_id -def listed(target, strict=True): +def listed(target): arguments = argparse.Namespace( repository=str(target), - worktrees_only=strict, scan_root=None, target_id=None, mode=None, @@ -182,6 +184,8 @@ if scenario == "identity": "serviceA": repository / "service-a", "worktreeServiceA": worktree / "service-a", "serviceB": repository / "service-b", + "mixedCase": repository / "MixedCase", + "worktreeMixedCase": worktree / "MixedCase", "clone": clone, } target_ids = { @@ -192,23 +196,9 @@ if scenario == "identity": name: repository_identity(target) for name, target in targets.items() } - common = os.path.normcase(os.path.realpath(git( + common = os.path.realpath(git( repository, "rev-parse", "--path-format=absolute", "--git-common-dir" - ))) - metadata = Path(common).stat() - device = serialize_filesystem_identity(metadata.st_dev) - inode = serialize_filesystem_identity(metadata.st_ino) - marker = Path(common) / "description" - generation = marker.lstat() - generation_device = serialize_filesystem_identity(generation.st_dev) - generation_inode = serialize_filesystem_identity(generation.st_ino) - expected = "repository_sha256_" + hashlib.sha256( - ( - f"git-common-dir\0{common}\0{device}\0{inode}\0" - f"git-description\0{generation_device}\0{generation_inode}\0" - f"{generation.st_ctime_ns}\0." - ).encode() - ).hexdigest() + )) forged_identity = repository_identity(forged_worktree()) (repository / "service-a" / "service.py").write_text("value = 2\n") git(repository, "add", ".") @@ -224,27 +214,68 @@ if scenario == "identity": "https://different-user:DIFFERENT_SYNTHETIC_PASSWORD@example.test/other/repo.git", ) remote_independent = repository_identity(repository) == identities["repository"] - saved_marker = root / "saved-description" - marker.rename(saved_marker) - missing_marker_identity = repository_identity(repository) + description = Path(common) / "description" + description.write_text("An ordinary user-edited Git description.\n") + description_edit_independent = repository_identity(repository) == identities["repository"] + os.chmod(description, 0o400) + description_mode_independent = repository_identity(repository) == identities["repository"] + os.chmod(description, 0o600) + description.unlink() + description_absence_independent = repository_identity(repository) == identities["repository"] + custom_template = root / "custom-template" + custom_template.mkdir() + git(custom_template, "init", "-q", "--template=") + markerless_repository_identity = repository_identity(custom_template) + with patch("workbench_target_state._repository_birth_time_ns", side_effect=(41, 42)): + recycled_inode_distinguished = ( + repository_identity(repository) != repository_identity(repository) + ) + original_normcase = os.path.normcase + os.path.normcase = lambda path: os.fspath(path).lower() try: - marker.symlink_to(saved_marker) - except (NotImplementedError, OSError): - symlink_supported = False - symlink_marker_identity = None - else: - symlink_supported = True - symlink_marker_identity = repository_identity(repository) + case_preserved = ( + repository_relative_path(repository / "MixedCase") == "MixedCase" + and repository_identity(repository / "MixedCase") == identities["mixedCase"] + ) + finally: + os.path.normcase = original_normcase + + legacy_metadata = SimpleNamespace(st_ctime_ns=41) + modern_metadata = SimpleNamespace(st_birthtime_ns=43, st_ctime_ns=41) + with patch.object(os, "name", "nt"): + windows_legacy_birth_time = _repository_birth_time_ns(common, legacy_metadata) + windows_modern_birth_time = _repository_birth_time_ns(common, modern_metadata) + linux_birth_times = {} + with patch.object(sys, "platform", "linux"), patch.object(os, "name", "posix"): + for label, output, status in ( + ("valid", "42.000000123\n", 0), + ("unavailable", "0.000000000\n", 0), + ("malformed", "42.123\n", 0), + ("failed", "42.000000123\n", 1), + ): + with patch("workbench_target_state.subprocess.run") as stat_command: + stat_command.return_value = SimpleNamespace(stdout=output, returncode=status) + linux_birth_times[label] = _repository_birth_time_ns(common, legacy_metadata) + if label == "valid": + linux_locale = stat_command.call_args.kwargs["env"]["LC_ALL"] + with patch("workbench_target_state.subprocess.run", side_effect=OSError("missing stat")): + linux_birth_times["missing"] = _repository_birth_time_ns(common, legacy_metadata) print(json.dumps({ "identities": identities, - "expected": expected, "forgedIdentity": forged_identity, "commitIndependent": commit_independent, "worktreeIndependent": worktree_independent, "remoteIndependent": remote_independent, - "missingMarkerIdentity": missing_marker_identity, - "symlinkSupported": symlink_supported, - "symlinkMarkerIdentity": symlink_marker_identity, + "descriptionEditIndependent": description_edit_independent, + "descriptionModeIndependent": description_mode_independent, + "descriptionAbsenceIndependent": description_absence_independent, + "markerlessRepositoryIdentity": markerless_repository_identity, + "recycledInodeDistinguished": recycled_inode_distinguished, + "casePreserved": case_preserved, + "windowsLegacyBirthTime": windows_legacy_birth_time, + "windowsModernBirthTime": windows_modern_birth_time, + "linuxBirthTimes": linux_birth_times, + "linuxLocale": linux_locale, "targetIdsPreserved": all( target_ids[name] == stable_target_id(target) for name, target in targets.items() @@ -267,17 +298,14 @@ elif scenario == "history": add_scan("spoof-a", clone / "service-a") add_scan("forged-root", forged_worktree()) before = { - "strictRoot": listed(repository), - "legacyRoot": listed(repository, strict=False), - "strictServiceA": listed(repository / "service-a"), - "legacyServiceA": listed(repository / "service-a", strict=False), - "strictServiceB": listed(repository / "service-b"), + "root": listed(repository), + "serviceA": listed(repository / "service-a"), + "serviceB": listed(repository / "service-b"), } git(repository, "worktree", "remove", "--force", str(worktree)) after = { - "strictRoot": listed(repository), - "legacyRoot": listed(repository, strict=False), - "strictServiceA": listed(repository / "service-a"), + "root": listed(repository), + "serviceA": listed(repository / "service-a"), } arguments = argparse.Namespace(repository=str(repository), force=False) matched = history.list_unmatched_scan_pairs( @@ -382,7 +410,6 @@ elif scenario == "replacement": git(worktree, "commit", "-qm", "replacement") git(worktree, "remote", "add", "origin", sys.argv[7]) before = listed(worktree) - regular_before = listed(worktree, strict=False) try: add_scan("replacement-owner", worktree) except SystemExit as error: @@ -437,7 +464,6 @@ elif scenario == "replacement": unverified_owner_error = None print(json.dumps({ "before": before, - "regularBefore": regular_before, "after": after, "registrationError": registration_error, "matchingError": matching_error, @@ -479,7 +505,6 @@ elif scenario == "git-replacement": and original_metadata.st_ino == replacement_metadata.st_ino ), "visibleScans": listed(repository), - "regularScans": listed(repository, strict=False), "registrationError": registration_error, })) @@ -488,15 +513,11 @@ elif scenario == "unverified-owner": add_scan("trusted-alias", worktree) git(repository, "worktree", "remove", "--force", str(worktree)) before = listed(repository) - regular_before = listed(repository, strict=False) add_scan("unverified-owner", repository, "missing") after = listed(repository) - regular_after = listed(repository, strict=False) print(json.dumps({ "before": before, - "regularBefore": regular_before, "after": after, - "regularAfter": regular_after, })) elif scenario == "nongit": @@ -563,9 +584,12 @@ describe("durable workbench repository identities", () => { const result = runProbe("identity", repositories); const identities = result["identities"] as Record; - expect(identities["repository"]).toBe(result["expected"] as string); + expect(identities["repository"]).toMatch( + /^repository_sha256_[a-f0-9]{64}$/, + ); expect(identities["worktree"]).toBe(identities["repository"]); expect(identities["worktreeServiceA"]).toBe(identities["serviceA"]); + expect(identities["worktreeMixedCase"]).toBe(identities["mixedCase"]); expect(identities["serviceA"]).not.toBe(identities["repository"]); expect(identities["serviceA"]).not.toBe(identities["serviceB"]); expect(identities["clone"]).not.toBe(identities["repository"]); @@ -573,8 +597,24 @@ describe("durable workbench repository identities", () => { expect(result["commitIndependent"]).toBe(true); expect(result["worktreeIndependent"]).toBe(true); expect(result["remoteIndependent"]).toBe(true); - expect(result["missingMarkerIdentity"]).toBeNull(); - expect(result["symlinkMarkerIdentity"]).toBeNull(); + expect(result["descriptionEditIndependent"]).toBe(true); + expect(result["descriptionModeIndependent"]).toBe(true); + expect(result["descriptionAbsenceIndependent"]).toBe(true); + expect(result["markerlessRepositoryIdentity"]).toMatch( + /^repository_sha256_[a-f0-9]{64}$/, + ); + expect(result["recycledInodeDistinguished"]).toBe(true); + expect(result["casePreserved"]).toBe(true); + expect(result["windowsLegacyBirthTime"]).toBe(41); + expect(result["windowsModernBirthTime"]).toBe(43); + expect(result["linuxBirthTimes"]).toEqual({ + valid: 42_000_000_123, + unavailable: null, + malformed: null, + failed: null, + missing: null, + }); + expect(result["linuxLocale"]).toBe("C"); expect(result["targetIdsPreserved"]).toBe(true); expect(JSON.stringify(result["stored"])).not.toContain( "SYNTHETIC_PASSWORD", @@ -586,26 +626,11 @@ describe("durable workbench repository identities", () => { const before = result["before"] as Record; const after = result["after"] as Record; - expect(before["strictRoot"]).toEqual(["canonical-root", "linked-root"]); - expect(before["legacyRoot"]).toEqual([ - "canonical-root", - "linked-root", - "spoof-root", - ]); - expect(before["strictServiceA"]).toEqual(["canonical-a", "linked-a"]); - expect(before["legacyServiceA"]).toEqual([ - "canonical-a", - "linked-a", - "spoof-a", - ]); - expect(before["strictServiceB"]).toEqual(["canonical-b"]); - expect(after["strictRoot"]).toEqual(["canonical-root", "linked-root"]); - expect(after["legacyRoot"]).toEqual([ - "canonical-root", - "linked-root", - "spoof-root", - ]); - expect(after["strictServiceA"]).toEqual(["canonical-a", "linked-a"]); + expect(before["root"]).toEqual(["canonical-root", "linked-root"]); + expect(before["serviceA"]).toEqual(["canonical-a", "linked-a"]); + expect(before["serviceB"]).toEqual(["canonical-b"]); + expect(after["root"]).toEqual(["canonical-root", "linked-root"]); + expect(after["serviceA"]).toEqual(["canonical-a", "linked-a"]); expect(result["matchedScanCount"]).toBe(3); expect(result["matchingBatches"]).toBe(2); expect(result["unavailableScans"]).toBe(1); @@ -629,7 +654,6 @@ describe("durable workbench repository identities", () => { const result = runProbe("replacement", fixture()); expect(result["before"]).toEqual([]); - expect(result["regularBefore"]).toEqual([]); expect(result["after"]).toEqual([]); expect(result["registrationError"]).toContain( "refusing to reuse its target", @@ -653,7 +677,6 @@ describe("durable workbench repository identities", () => { expect(result["checkoutOwnerUnchanged"]).toBe(true); expect(result["replacementIdentity"]).not.toBe(result["originalIdentity"]); expect(result["visibleScans"]).toEqual([]); - expect(result["regularScans"]).toEqual([]); expect(result["registrationError"]).toContain( "refusing to reuse its target", ); @@ -663,9 +686,7 @@ describe("durable workbench repository identities", () => { const result = runProbe("unverified-owner", fixture()); expect(result["before"]).toEqual(["trusted-alias"]); - expect(result["regularBefore"]).toEqual(["trusted-alias"]); expect(result["after"]).toEqual(["unverified-owner"]); - expect(result["regularAfter"]).toEqual(["unverified-owner"]); }, 30_000); test("keeps non-Git paths isolated and never equates unrelated null target IDs", () => { From e895dde33894cb6dedbc1045e0cccdab27deffd4 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 15 Aug 2026 13:01:30 -0700 Subject: [PATCH 04/21] fix(workbench): preserve recreated repository subdirectories --- .../scripts/workbench_scan_history.py | 55 ++++---- .../scripts/workbench_target_state.py | 22 +++- .../workbench-repository-identity.test.ts | 124 ++++++++++++++++++ 3 files changed, 170 insertions(+), 31 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py index 37b93f17e..66421233e 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py @@ -180,15 +180,38 @@ def list_scans( else [] ) repository_clauses = ( - [_owned_scan_clause(repository, check_ownership, values)] + [ + _owned_scan_clause( + repository, + check_ownership + and not ( + identity_column + and requested_repository["repository_identity"] is not None + and requested_repository["repository_identity"] == live_identity + and ownership_matches + and repository_relative_path(repository) not in (None, ".") + ), + values, + ) + ] if identity_matches else ["0"] ) for target in related_targets: + target_path = Path(target["target_path"]) + target_identity = target["repository_identity"] if identity_column else None + verified_target = ( + check_ownership + and target_identity is not None + and repository_relative_path(target_path) not in (None, ".") + and _repository_ownership_matches( + connection, target_path, target["target_id"], target_identity + ) + ) repository_clauses.append( _owned_scan_clause( - Path(target["target_path"]), - check_ownership, + target_path, + check_ownership and not verified_target, values, target_id=target["target_id"], ) @@ -346,29 +369,11 @@ def _repository_ownership_matches( stored_identity: str | None = None, ) -> bool: try: - metadata = repository.stat() - except OSError: + repository.stat() + _require_current_target_owner(connection, target_id or "", str(repository), stored_identity) + except (OSError, SystemExit): return False - scans = connection.execute( - """ - SELECT target_device, target_inode - FROM scans - WHERE target_path = ? OR target_id = ? - """, - (str(repository), target_id), - ) - current_device = serialize_filesystem_identity(metadata.st_dev) - current_inode = serialize_filesystem_identity(metadata.st_ino) - historical_scan = False - verified_owner = False - for scan in scans: - historical_scan = True - if scan["target_device"] is None and scan["target_inode"] is None: - continue - if scan["target_device"] != current_device or scan["target_inode"] != current_inode: - return False - verified_owner = True - return not historical_scan or verified_owner or stored_identity is None + return True def list_unmatched_scan_pairs( diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index 538958088..6f1fd81c2 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -186,25 +186,35 @@ def _require_current_target_owner( (target_id, target_path), ) historical_scan = False - verified_owner = False + recorded_owner = False + malformed_owner = False mismatch = False for scan in scans: historical_scan = True device, inode = scan["target_device"], scan["target_inode"] if device is None and inode is None: continue + if device is None or inode is None: + malformed_owner = True + break + recorded_owner = True if not stored_filesystem_identity_matches( device, metadata.st_dev ) or not stored_filesystem_identity_matches(inode, metadata.st_ino): mismatch = True - break - verified_owner = True + verified_repository = ( + stored_identity is not None and repository_identity(target) == stored_identity + ) if ( - mismatch + malformed_owner + or mismatch + and ( + not verified_repository + or repository_relative_path(target) in (None, ".") + ) or stored_identity is not None and ( - repository_identity(target) != stored_identity - or historical_scan and not verified_owner + not verified_repository or historical_scan and not recorded_owner ) ): raise SystemExit( diff --git a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts index 5a7b382bf..a87bc3005 100644 --- a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts +++ b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts @@ -81,6 +81,7 @@ sys.path.insert(0, sys.argv[1]) import workbench_scan_history as history from filesystem_identity import serialize_filesystem_identity +from workbench_native_indexes import repository_target_ids from workbench_schema import MIGRATIONS, apply_migrations from workbench_target_state import ( _repository_birth_time_ns, @@ -475,6 +476,95 @@ elif scenario == "replacement": ).fetchone() is not None, })) +elif scenario == "recreated-directory": + target = repository / "service-a" + linked = worktree / "service-a" + target_id = add_scan("before-recreation", target) + linked_id = add_scan("linked-scope", linked) + original_metadata = target.stat() + original_identity = repository_identity(target) + original_scan = connection.execute( + "SELECT target_device, target_inode FROM scans WHERE id = 'before-recreation'" + ).fetchone() + + shutil.rmtree(target) + (root / "retired-directory-inode").mkdir() + target.mkdir() + (target / "service.py").write_text("value = 2\n") + recreated_metadata = target.stat() + before_rescan = listed(target) + before_aliases = repository_target_ids(connection, target_id) + recreated_id = add_scan("after-recreation", target) + repeated_id = add_scan("repeated-rescan", target) + after_rescan = listed(target) + after_aliases = repository_target_ids(connection, target_id) + + malformed_target = repository / "service-b" + add_scan("malformed-owner", malformed_target, "malformed") + try: + ensure_security_target(connection, str(malformed_target)) + except SystemExit as error: + malformed_owner_error = str(error) + else: + malformed_owner_error = None + + plain = root / "recreated-nongit" + plain.mkdir() + add_scan("plain-before-recreation", plain) + shutil.rmtree(plain) + (root / "retired-nongit-inode").mkdir() + plain.mkdir() + try: + ensure_security_target(connection, str(plain)) + except SystemExit as error: + nongit_owner_error = str(error) + else: + nongit_owner_error = None + + root_target_id = add_scan("linked-root-before", worktree) + add_scan("canonical-root-alias", repository) + root_identity = repository_identity(worktree) + root_metadata = worktree.stat() + git_pointer = (worktree / ".git").read_text() + shutil.rmtree(worktree) + (root / "retired-worktree-root-inode").mkdir() + worktree.mkdir() + (worktree / ".git").write_text(git_pointer) + try: + ensure_security_target(connection, str(worktree)) + except SystemExit as error: + recreated_root_error = str(error) + else: + recreated_root_error = None + + preserved_scan = connection.execute( + "SELECT target_device, target_inode FROM scans WHERE id = 'before-recreation'" + ).fetchone() + print(json.dumps({ + "ownerChanged": ( + original_metadata.st_dev != recreated_metadata.st_dev + or original_metadata.st_ino != recreated_metadata.st_ino + ), + "identityPreserved": repository_identity(target) == original_identity, + "targetIdPreserved": recreated_id == target_id and repeated_id == target_id, + "historicalOwnerPreserved": ( + preserved_scan["target_device"] == original_scan["target_device"] + and preserved_scan["target_inode"] == original_scan["target_inode"] + ), + "beforeRescan": before_rescan, + "afterRescan": after_rescan, + "beforeAliases": sorted(before_aliases), + "afterAliases": sorted(after_aliases), + "expectedAliases": sorted((target_id, linked_id)), + "malformedOwnerError": malformed_owner_error, + "nongitOwnerError": nongit_owner_error, + "rootOwnerChanged": root_metadata.st_ino != worktree.stat().st_ino, + "rootIdentityPreserved": repository_identity(worktree) == root_identity, + "recreatedRootError": recreated_root_error, + "recreatedRootScans": listed(worktree), + "recreatedRootAliases": sorted(repository_target_ids(connection, root_target_id)), + })) + elif scenario == "git-replacement": add_scan("original-repository", repository) add_scan("original-alias", worktree) @@ -682,6 +772,40 @@ describe("durable workbench repository identities", () => { ); }, 30_000); + test("rescans recreated directories when their Git repository and scope are unchanged", () => { + const result = runProbe("recreated-directory", fixture()); + + expect(result["ownerChanged"]).toBe(true); + expect(result["identityPreserved"]).toBe(true); + expect(result["targetIdPreserved"]).toBe(true); + expect(result["historicalOwnerPreserved"]).toBe(true); + expect(result["beforeRescan"]).toEqual([ + "before-recreation", + "linked-scope", + ]); + expect(result["afterRescan"]).toEqual([ + "after-recreation", + "before-recreation", + "linked-scope", + "repeated-rescan", + ]); + expect(result["beforeAliases"]).toEqual(result["expectedAliases"]); + expect(result["afterAliases"]).toEqual(result["expectedAliases"]); + expect(result["malformedOwnerError"]).toContain( + "refusing to reuse its target", + ); + expect(result["nongitOwnerError"]).toContain( + "refusing to reuse its target", + ); + expect(result["rootOwnerChanged"]).toBe(true); + expect(result["rootIdentityPreserved"]).toBe(true); + expect(result["recreatedRootError"]).toContain( + "refusing to reuse its target", + ); + expect(result["recreatedRootScans"]).toEqual([]); + expect(result["recreatedRootAliases"]).toEqual([]); + }, 30_000); + test("does not expand trusted aliases when historical checkout ownership is unverified", () => { const result = runProbe("unverified-owner", fixture()); From 628316986c1da26c340d901ba2cd438d0c5c9e52 Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sat, 15 Aug 2026 23:11:29 -0700 Subject: [PATCH 05/21] fix(workbench): refresh the bundled history plugin Advance the paired standalone plugin version so existing staged marketplaces load the linked-worktree history helpers. Leave the npm package release version unchanged. --- sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json | 2 +- sdk/typescript/src/version.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json b/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json index 04c30b9c6..4015ea149 100644 --- a/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json +++ b/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "codex-security", - "version": "0.1.20", + "version": "0.1.21", "description": "Codex Security workflows for security scans, analysis, and investigation.", "author": { "name": "OpenAI" diff --git a/sdk/typescript/src/version.ts b/sdk/typescript/src/version.ts index 01dd50067..955feef3f 100644 --- a/sdk/typescript/src/version.ts +++ b/sdk/typescript/src/version.ts @@ -8,7 +8,7 @@ const PACKAGE_VERSIONS = packageVersions( export const VERSION = PACKAGE_VERSIONS.package; export const CODEX_SDK_VERSION = PACKAGE_VERSIONS.sdk; export const CODEX_EXECUTABLE_VERSION = PACKAGE_VERSIONS.executable; -export const BUNDLED_PLUGIN_VERSION = "0.1.20" as const; +export const BUNDLED_PLUGIN_VERSION = "0.1.21" as const; const PACKAGE_NAME = "@openai/codex-security"; const VERSION_PATTERN = From 088033dde3e540359ac08fed5a8e666719378a7b Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sat, 15 Aug 2026 23:25:18 -0700 Subject: [PATCH 06/21] fix(workbench): verify saved history candidates Use recorded checkout ownership before falling back to live repository identities. Preserve matching persisted identities for removed worktrees and retain verified same-origin comparisons. --- .../scripts/workbench_scan_history.py | 64 +++++++++------ .../scripts/workbench_target_state.py | 22 ++++-- .../workbench-canonical-paths.test.ts | 11 +-- .../workbench-repository-identity.test.ts | 78 ++++++++++++++++++- 4 files changed, 140 insertions(+), 35 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py index c2cccd951..565043975 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py @@ -22,22 +22,20 @@ repository_identity, repository_relative_path, supports_repository_identity, + verified_repository_identity, ) def _same_repository( + connection: sqlite3.Connection, before: sqlite3.Row, after: sqlite3.Row, *, after_identity: tuple[str | None, tuple[str, str] | None, str | None] | None = None, ) -> bool: before_target_id = before["target_id"] - if before_target_id and before_target_id == after["target_id"]: - return True before_target = Path(before["target_path"]) after_target = Path(after["target_path"]) - if str(before_target.resolve()) == str(after_target.resolve()): - return True before_stored_identity = ( before["repository_identity"] if "repository_identity" in before.keys() else None ) @@ -51,12 +49,29 @@ def _same_repository( if after_identity is not None else None ) - if before_stored_identity and before_stored_identity == after_grouping_identity: + if before_stored_identity is not None and before_stored_identity == after_grouping_identity: return True + if ( + before_target_id and before_target_id == after["target_id"] + or str(before_target.resolve()) == str(after_target.resolve()) + ): + return before_stored_identity is None or after_grouping_identity is None - before_live_identity = repository_identity(before_target) + before_live_identity = verified_repository_identity( + connection, + before_target_id or "", + str(before_target), + stored_identity=before_stored_identity, + ) after_live_identity = ( - repository_identity(after_target) if after_identity is None else after_identity[0] + verified_repository_identity( + connection, + after["target_id"] or "", + str(after_target), + stored_identity=after_stored_identity, + ) + if after_identity is None + else after_identity[0] ) if before_live_identity is not None and before_live_identity == after_live_identity: return True @@ -147,15 +162,16 @@ def list_scans( ) ownership_matches = ownership_matches and identity_matches requested_identity = ( - ( - requested_repository["repository_identity"] - if ( - identity_column - and ownership_matches - and requested_repository["repository_identity"] is not None - ) - else live_identity - ), + verified_repository_identity( + connection, + requested_repository["target_id"], + str(repository), + stored_identity=( + requested_repository["repository_identity"] if identity_column else None + ), + ) + if ownership_matches + else None, None, None, ) @@ -169,6 +185,7 @@ def list_scans( else "SELECT id AS target_id, current_path AS target_path FROM security_targets" ) if _same_repository( + connection, target, requested_repository, after_identity=requested_identity, @@ -416,11 +433,11 @@ def list_unmatched_scan_pairs( if identity_column else "SELECT * FROM scans WHERE status = 'complete' ORDER BY started_at, id" ) - live_identity = repository_identity(repository) - requested_identity = ( - requested["repository_identity"] - if identity_column and requested["repository_identity"] is not None - else live_identity + requested_identity = verified_repository_identity( + connection, + requested["target_id"], + str(repository), + stored_identity=requested["repository_identity"] if identity_column else None, ) requested_group = ( requested_identity, @@ -432,6 +449,7 @@ def list_unmatched_scan_pairs( for scan in connection.execute(scan_query) if str(Path(scan["target_path"]).resolve()) == str(repository) or _same_repository( + connection, scan, requested, after_identity=requested_group, @@ -511,7 +529,7 @@ def compare_scans( raise SystemExit("Select two different scans to compare.") if before["status"] != "complete" or after["status"] != "complete": raise SystemExit("Only completed scans can be compared.") - if not _same_repository(before, after): + if not _same_repository(connection, before, after): raise SystemExit("Semantic scan comparisons require the same repository target.") cached = connection.execute( "SELECT result_json FROM scan_comparisons WHERE before_scan_id = ? AND after_scan_id = ?", @@ -655,7 +673,7 @@ def save_scan_comparison( raise SystemExit("Select two different scans to compare.") if before["status"] != "complete" or after["status"] != "complete": raise SystemExit("Only completed scans can be compared.") - if not _same_repository(before, after): + if not _same_repository(connection, before, after): raise SystemExit("Semantic scan comparisons require the same repository target.") read_coverage(after) before_findings = _scan_findings(connection, before["id"]) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index 5870d122f..198c02d19 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -123,8 +123,12 @@ def supports_repository_identity(connection: sqlite3.Connection) -> bool: ) -def _verified_repository_identity( - connection: sqlite3.Connection, target_id: str, target_path: str +def verified_repository_identity( + connection: sqlite3.Connection, + target_id: str, + target_path: str, + *, + stored_identity: str | None = None, ) -> str | None: target = Path(target_path) try: @@ -132,6 +136,14 @@ def _verified_repository_identity( except OSError: return None + if stored_identity is not None: + try: + _require_current_target_owner(connection, target_id, target_path, stored_identity) + except SystemExit: + return None + identity = repository_identity(target) + return identity if identity == stored_identity else None + scan_columns = { row["name"] for row in connection.execute("PRAGMA table_info(scans)") } @@ -234,7 +246,7 @@ def backfill_repository_identities(connection: sqlite3.Connection) -> None: """ ).fetchall() for target in targets: - identity = _verified_repository_identity( + identity = verified_repository_identity( connection, str(target["id"]), target["current_path"] ) if identity is not None: @@ -287,7 +299,7 @@ def ensure_security_target( connection, target_id, target_path, existing["repository_identity"] ) if supports_identity and existing["repository_identity"] is None: - identity = _verified_repository_identity(connection, target_id, target_path) + identity = verified_repository_identity(connection, target_id, target_path) if identity is not None: connection.execute( """ @@ -301,7 +313,7 @@ def ensure_security_target( target_id = stable_target_id(Path(target_path)) timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") if supports_identity: - identity = _verified_repository_identity(connection, target_id, target_path) + identity = verified_repository_identity(connection, target_id, target_path) connection.execute( """ INSERT OR IGNORE INTO security_targets ( diff --git a/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts b/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts index 0d64d4bf4..cfc527422 100644 --- a/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts +++ b/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts @@ -83,6 +83,7 @@ const caseSensitiveWindowsHistoryProbe = [ "targets = [upper_root, lower_root, upper_scope, lower_scope, linked_scope, clone_scope]", "identities = {entry['target_path']: entry['repository_identity'] for entry in targets}", "history.repository_identity = lambda path: identities.get(str(path))", + "history.verified_repository_identity = lambda connection, target_id, path, **kwargs: identities.get(str(path))", "history.repository_relative_path = lambda path: WindowsPath(path).name", "history._repository_origin = lambda path: ('example.test', 'synthetic/repository')", "root_group = ('root-upper', None, None)", @@ -90,11 +91,11 @@ const caseSensitiveWindowsHistoryProbe = [ "checks = {", " 'nativeWindowsScopeEquality': WindowsPath(upper_scope['target_path']) == WindowsPath(lower_scope['target_path']),", " 'nativeWindowsRootEquality': WindowsPath(upper_root['target_path']) == WindowsPath(lower_root['target_path']),", - " 'caseSensitiveScopesMatch': history._same_repository(lower_scope, upper_scope, after_identity=scope_group),", - " 'caseSensitiveRootsMatch': history._same_repository(lower_root, upper_root, after_identity=root_group),", - " 'linkedWorktreeMatches': history._same_repository(linked_scope, upper_scope, after_identity=scope_group),", - " 'sameOriginCloneMatches': history._same_repository(clone_scope, upper_scope, after_identity=scope_group),", - " 'exactResolvedPathMatches': history._same_repository(same_path, upper_scope, after_identity=scope_group),", + " 'caseSensitiveScopesMatch': history._same_repository(connection, lower_scope, upper_scope, after_identity=scope_group),", + " 'caseSensitiveRootsMatch': history._same_repository(connection, lower_root, upper_root, after_identity=root_group),", + " 'linkedWorktreeMatches': history._same_repository(connection, linked_scope, upper_scope, after_identity=scope_group),", + " 'sameOriginCloneMatches': history._same_repository(connection, clone_scope, upper_scope, after_identity=scope_group),", + " 'exactResolvedPathMatches': history._same_repository(connection, same_path, upper_scope, after_identity=scope_group),", "}", "for entry in (upper_scope, lower_scope):", " connection.execute('INSERT INTO security_targets VALUES (?, ?, ?)', (entry['target_id'], entry['target_path'], entry['repository_identity']))", diff --git a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts index a87bc3005..186b08f48 100644 --- a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts +++ b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts @@ -565,6 +565,62 @@ elif scenario == "recreated-directory": "recreatedRootAliases": sorted(repository_target_ids(connection, root_target_id)), })) +elif scenario == "candidate-generation": + add_scan("requested-scan", repository) + legacy_id = add_scan("trusted-alias", worktree) + candidate_id = add_scan("previous-generation", clone) + requested_identity = repository_identity(repository) + candidate_identity = repository_identity(clone) + + def live_identity(target): + return requested_identity if Path(target) == clone else repository_identity(target) + + def automatic_history(): + matching = history.list_unmatched_scan_pairs( + connection, + argparse.Namespace(repository=str(repository), force=False), + backfill_finding_details=lambda _connection, _scan: None, + read_coverage=lambda _scan: {}, + ) + matching_ids = { + batch["afterScanId"] for batch in matching["batches"] + } | { + scan["scanId"] + for batch in matching["batches"] + for scan in batch["beforeScans"] + } + return { + "scans": listed(repository), + "matchingScanCount": matching["scanCount"], + "matchingScanIds": sorted(matching_ids), + } + + with ( + patch("workbench_target_state.repository_identity", side_effect=live_identity), + patch("workbench_scan_history.repository_identity", side_effect=live_identity), + ): + persisted = automatic_history() + connection.execute( + "UPDATE security_targets SET repository_identity = NULL WHERE id = ?", + (legacy_id,), + ) + verified_legacy = automatic_history() + connection.execute( + "UPDATE scans SET target_device = NULL, target_inode = NULL WHERE id = ?", + ("trusted-alias",), + ) + unverified_legacy = automatic_history() + + print(json.dumps({ + "persisted": persisted, + "verifiedLegacy": verified_legacy, + "unverifiedLegacy": unverified_legacy, + "candidateIdentityPreserved": connection.execute( + "SELECT repository_identity FROM security_targets WHERE id = ?", + (candidate_id,), + ).fetchone()[0] == candidate_identity, + })) + elif scenario == "git-replacement": add_scan("original-repository", repository) add_scan("original-alias", worktree) @@ -632,8 +688,8 @@ elif scenario == "nongit": print(json.dumps({ "firstIdentity": repository_identity(first), "secondIdentity": repository_identity(second), - "differentNullIdsMatch": history._same_repository(before, after), - "sameNullPathMatches": history._same_repository(before, same), + "differentNullIdsMatch": history._same_repository(connection, before, after), + "sameNullPathMatches": history._same_repository(connection, before, same), "firstScans": listed(first), "secondScans": listed(second), })) @@ -772,6 +828,24 @@ describe("durable workbench repository identities", () => { ); }, 30_000); + test("does not discover historical targets through a conflicting live generation", () => { + const result = runProbe("candidate-generation", fixture()); + const verified = { + scans: ["requested-scan", "trusted-alias"], + matchingScanCount: 2, + matchingScanIds: ["requested-scan", "trusted-alias"], + }; + + expect(result["persisted"]).toEqual(verified); + expect(result["verifiedLegacy"]).toEqual(verified); + expect(result["unverifiedLegacy"]).toEqual({ + scans: ["requested-scan"], + matchingScanCount: 1, + matchingScanIds: [], + }); + expect(result["candidateIdentityPreserved"]).toBe(true); + }, 30_000); + test("rescans recreated directories when their Git repository and scope are unchanged", () => { const result = runProbe("recreated-directory", fixture()); From b4b5c3a6de76b45504324dea99410802ba88e010 Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sun, 16 Aug 2026 00:40:18 -0700 Subject: [PATCH 07/21] fix(workbench): unify verified repository history --- .../scripts/workbench_feedback.py | 8 +- .../scripts/workbench_native_indexes.py | 121 ++---- .../scripts/workbench_scan_history.py | 305 +++---------- .../scripts/workbench_schema.py | 3 + .../scripts/workbench_target_state.py | 411 ++++++++++++++---- .../tests-ts/repository-findings.test.ts | 18 + .../workbench-canonical-paths.test.ts | 29 +- .../tests-ts/workbench-identity-cache.test.ts | 371 ++++++++++++++++ .../workbench-repository-identity.test.ts | 10 +- 9 files changed, 835 insertions(+), 441 deletions(-) create mode 100644 sdk/typescript/tests-ts/workbench-identity-cache.test.ts diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py b/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py index ac3e1651c..e92ec96f6 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py @@ -16,18 +16,20 @@ FINDING_SUMMARY_BYTES, FINDING_TITLE_BYTES, ) -from workbench_native_indexes import _indexed_findings, repository_target_ids +from workbench_native_indexes import _indexed_findings +from workbench_target_state import RepositoryIdentityCache from workbench_validation import bounded_output_text def get_scan_feedback(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict[str, Any]: - target_ids = sorted(repository_target_ids(connection, scan["target_id"])) + identities = RepositoryIdentityCache(connection) + target_ids = sorted(identities.target_ids(scan["target_id"])) if not target_ids: return {"scanId": scan["id"], "targetId": scan["target_id"], "falsePositives": []} indexed_findings = { finding_id: finding - for finding in _indexed_findings(connection) + for finding in _indexed_findings(connection, identities=identities) if finding["target_id"] in target_ids for finding_id in finding["matched_finding_ids"] } diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py index 67e44a17b..0f16dbba7 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py @@ -13,43 +13,17 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import workbench_scan_history as scan_history from workbench_constants import FINDING_SUMMARY_BYTES, FINDING_TITLE_BYTES, FINDINGS_PAGE_MAX -from workbench_target_state import _require_current_target_owner, supports_repository_identity +from workbench_target_state import RepositoryIdentityCache from workbench_validation import bounded_output_text -def repository_target_ids(connection: sqlite3.Connection, target_id: str) -> set[str]: - if not supports_repository_identity(connection): - return {target_id} - - requested_target = connection.execute( - "SELECT current_path, repository_identity FROM security_targets WHERE id = ?", - (target_id,), - ).fetchone() - if requested_target is None: - return {target_id} - try: - _require_current_target_owner( - connection, - target_id, - requested_target["current_path"], - requested_target["repository_identity"], - ) - except SystemExit: - return set() - - rows = connection.execute( - """ - SELECT id - FROM security_targets - WHERE id = ? - OR ( - repository_identity IS NOT NULL - AND repository_identity = ? - ) - """, - (target_id, requested_target["repository_identity"]), - ) - return {row["id"] for row in rows} or {target_id} +def repository_target_ids( + connection: sqlite3.Connection, + target_id: str, + *, + identities: RepositoryIdentityCache | None = None, +) -> set[str]: + return (identities or RepositoryIdentityCache(connection)).target_ids(target_id) def list_global_findings( @@ -58,12 +32,13 @@ def list_global_findings( ) -> dict[str, Any]: limit = min(args.limit, FINDINGS_PAGE_MAX) query = args.query.strip().casefold() if args.query else "" + identities = RepositoryIdentityCache(connection) target_ids = ( - None if args.target_id is None else repository_target_ids(connection, args.target_id) + None if args.target_id is None else identities.target_ids(args.target_id) ) findings = ( row - for row in _indexed_findings(connection) + for row in _indexed_findings(connection, identities=identities) if (target_ids is None or row["target_id"] in target_ids) and (args.severity is None or row["severity"] == args.severity) and (args.status is None or row["status"] == args.status) @@ -113,27 +88,15 @@ def list_global_findings( } -def _indexed_findings(connection: sqlite3.Connection) -> Iterator[dict[str, Any]]: +def _indexed_findings( + connection: sqlite3.Connection, + *, + identities: RepositoryIdentityCache | None = None, +) -> Iterator[dict[str, Any]]: + identities = identities or RepositoryIdentityCache(connection) parents: dict[ tuple[tuple[str, str], str], tuple[tuple[str, str], str] ] = {} - has_repository_identities = supports_repository_identity(connection) - identity_column = "targets.repository_identity" if has_repository_identities else "NULL" - before_identity_column = ( - "before_targets.repository_identity" if has_repository_identities else "NULL" - ) - after_identity_column = ( - "after_targets.repository_identity" if has_repository_identities else "NULL" - ) - alias_condition = ( - "before_targets.repository_identity IS NOT NULL " - "AND before_targets.repository_identity = after_targets.repository_identity" - if has_repository_identities - else "0" - ) - - def repository_identity(target_id: str, identity: str | None) -> tuple[str, str]: - return ("target", target_id) if identity is None else ("repository", identity) def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: while identity in parents: @@ -141,11 +104,9 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: return identity for match in connection.execute( - f""" + """ SELECT before_scans.target_id AS before_target_id, after_scans.target_id AS after_target_id, - {before_identity_column} AS before_repository_identity, - {after_identity_column} AS after_repository_identity, before.finding_id AS before_finding_id, after.finding_id AS after_finding_id FROM scan_comparison_matches AS matches @@ -155,22 +116,21 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: JOIN finding_occurrences AS after ON after.id = matches.after_occurrence_id JOIN scans AS after_scans ON after_scans.id = after.scan_id JOIN security_targets AS after_targets ON after_targets.id = after_scans.target_id - WHERE before_scans.target_id = after_scans.target_id OR ({alias_condition}) """ ): + before_repository = identities.group(match["before_target_id"]) + after_repository = identities.group(match["after_target_id"]) + if before_repository != after_repository: + continue before = group( ( - repository_identity( - match["before_target_id"], match["before_repository_identity"] - ), + before_repository, match["before_finding_id"], ) ) after = group( ( - repository_identity( - match["after_target_id"], match["after_repository_identity"] - ), + after_repository, match["after_finding_id"], ) ) @@ -178,10 +138,10 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: parents[after] = before latest_scan_by_repository = { - repository_identity(row["target_id"], row["repository_identity"]): row["id"] + identities.group(row["target_id"]): row["id"] for row in connection.execute( - f""" - SELECT scans.target_id, scans.id, {identity_column} AS repository_identity + """ + SELECT scans.target_id, scans.id FROM scans JOIN security_targets AS targets ON targets.id = scans.target_id WHERE scans.status = 'complete' @@ -192,7 +152,7 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: grouped: dict[tuple[tuple[str, str], str], list[sqlite3.Row]] = {} for row in connection.execute( - f""" + """ SELECT occurrences.id AS occurrence_id, occurrences.finding_id, @@ -202,7 +162,6 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: scans.started_at AS scan_started_at, scans.target_id, targets.current_path AS target_path, - {identity_column} AS repository_identity, scans.scope, MAX(scans.updated_at, COALESCE(triage.updated_at, '')) AS updated_at, triage.status AS decision_status, @@ -228,7 +187,7 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: grouped.setdefault( group( ( - repository_identity(row["target_id"], row["repository_identity"]), + identities.group(row["target_id"]), row["finding_id"], ) ), @@ -251,14 +210,14 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: ): status = "open" scans = sorted({(row["scan_started_at"], row["scan_id"]) for row in occurrences}) + latest_scan_id = latest_scan_by_repository.get(identities.group(latest["target_id"])) findings.append( { **dict(latest), "close_reason": decision["close_reason"] if decision is not None else None, - "confirmed_in_latest_scan": latest_scan_by_repository.get( - repository_identity(latest["target_id"], latest["repository_identity"]) - ) - == latest["scan_id"], + "confirmed_in_latest_scan": any( + row["scan_id"] == latest_scan_id for row in occurrences + ), "known_since": scans[0][0], "known_scan_ids": [scan_id for _, scan_id in scans], "matched_finding_ids": sorted({row["finding_id"] for row in occurrences}), @@ -288,6 +247,7 @@ def list_repositories( args: argparse.Namespace | None = None, ) -> dict[str, Any]: scans = scan_history.list_scans(connection)["scans"] + identities = RepositoryIdentityCache(connection) scans_by_id = {scan["scanId"]: scan for scan in scans} scan_count_by_target: dict[str, int] = {} for scan in scans: @@ -302,16 +262,9 @@ def list_repositories( targets = {row["id"]: row for row in connection.execute("SELECT * FROM security_targets")} - def repository_group(target_id: str) -> tuple[str, str]: - target = targets[target_id] - identity = ( - target["repository_identity"] if "repository_identity" in target.keys() else None - ) - return ("target", target_id) if identity is None else ("repository", identity) - open_findings_by_repository = Counter( - repository_group(row["target_id"]) - for row in _indexed_findings(connection) + identities.group(row["target_id"]) + for row in _indexed_findings(connection, identities=identities) if row["status"] == "open" ) repositories = [ @@ -320,7 +273,7 @@ def repository_group(target_id: str) -> tuple[str, str]: "displayName": target["display_name"], "latestScan": latest_scan, "openFindingsCount": open_findings_by_repository.get( - repository_group(target_id), 0 + identities.group(target_id), 0 ), "scanCount": scan_count_by_target[target_id], "targetId": target_id, diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py index 565043975..bc220b678 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py @@ -8,7 +8,6 @@ import sys from pathlib import Path, PurePosixPath from typing import Any, Callable -from urllib.parse import urlsplit # Some plugin hosts launch Python with safe-path isolation enabled. sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -16,13 +15,9 @@ from report_projection import SEVERITY_ORDER from workbench_constants import FINDINGS_PAGE_MAX from workbench_scan_usage import stored_scan_cost_fields -from workbench_target import git_output from workbench_target_state import ( - _require_current_target_owner, - repository_identity, - repository_relative_path, + RepositoryIdentityCache, supports_repository_identity, - verified_repository_identity, ) @@ -31,90 +26,40 @@ def _same_repository( before: sqlite3.Row, after: sqlite3.Row, *, - after_identity: tuple[str | None, tuple[str, str] | None, str | None] | None = None, + identities: RepositoryIdentityCache | None = None, ) -> bool: - before_target_id = before["target_id"] - before_target = Path(before["target_path"]) - after_target = Path(after["target_path"]) - before_stored_identity = ( + identities = identities or RepositoryIdentityCache(connection) + before_stored = ( before["repository_identity"] if "repository_identity" in before.keys() else None ) - after_stored_identity = ( + after_stored = ( after["repository_identity"] if "repository_identity" in after.keys() else None ) - after_grouping_identity = ( - after_stored_identity - if after_stored_identity is not None - else after_identity[0] - if after_identity is not None - else None - ) - if before_stored_identity is not None and before_stored_identity == after_grouping_identity: + before_state = identities.for_row(before) + after_state = identities.for_row(after) + if before_state.resolved_path is None or after_state.resolved_path is None: + return False + before_group = before_stored or before_state.verified_identity + after_group = after_stored or after_state.verified_identity + if before_group is not None and before_group == after_group: return True if ( - before_target_id and before_target_id == after["target_id"] - or str(before_target.resolve()) == str(after_target.resolve()) + before_state.target_id and before_state.target_id == after_state.target_id + or before_state.resolved_path == after_state.resolved_path ): - return before_stored_identity is None or after_grouping_identity is None - - before_live_identity = verified_repository_identity( - connection, - before_target_id or "", - str(before_target), - stored_identity=before_stored_identity, - ) - after_live_identity = ( - verified_repository_identity( - connection, - after["target_id"] or "", - str(after_target), - stored_identity=after_stored_identity, - ) - if after_identity is None - else after_identity[0] - ) - if before_live_identity is not None and before_live_identity == after_live_identity: + return before_stored is None or after_stored is None + before_identity = before_state.verified_identity + after_identity = after_state.verified_identity + if before_identity is not None and before_identity == after_identity: return True - if before_live_identity is None or after_live_identity is None: + if before_identity is None or after_identity is None: return False - - after_origin = ( - _repository_origin(after_target) if after_identity is None else after_identity[1] - ) - if after_origin is None or _repository_origin(before_target) != after_origin: - return False - before_relative = repository_relative_path(before_target) - after_relative = ( - repository_relative_path(after_target) if after_identity is None else after_identity[2] + after_origin = identities.origin(after_state) + return ( + after_origin is not None + and identities.origin(before_state) == after_origin + and before_state.repository.relative_path == after_state.repository.relative_path ) - return before_relative is not None and before_relative == after_relative - - -def _repository_origin(target: Path) -> tuple[str, str] | None: - remote = git_output(target, "remote", "get-url", "origin") - if remote is None: - return None - if "://" in remote: - try: - parsed = urlsplit(remote) - port = parsed.port - except ValueError: - return None - if parsed.scheme not in {"https", "ssh"} or parsed.hostname is None: - return None - if parsed.query or parsed.fragment: - return None - host = parsed.hostname - if port is not None and port != {"https": 443, "ssh": 22}[parsed.scheme]: - host = f"{host}:{port}" - path = parsed.path - else: - authority, separator, path = remote.partition(":") - if not separator or "?" in path or "#" in path: - return None - host = authority.rsplit("@", 1)[-1] - path = path.strip("/").removesuffix(".git") - return (host.lower(), path) if host and path else None def list_scans( @@ -124,113 +69,42 @@ def list_scans( values: list[Any] = [] if args is not None and args.repository: repository = Path(args.repository).expanduser().resolve() - identity_column = supports_repository_identity(connection) - requested_identity_column = ( - ", (SELECT repository_identity FROM security_targets " - "WHERE current_path = ?) AS repository_identity" - if identity_column - else "" - ) - requested_values = ( - (str(repository), str(repository), str(repository)) - if identity_column - else (str(repository), str(repository)) - ) - requested_repository = connection.execute( - f""" - SELECT COALESCE((SELECT id FROM security_targets WHERE current_path = ?), '') AS target_id, - ? AS target_path - {requested_identity_column} - """, - requested_values, - ).fetchone() - scan_columns = { - row["name"] for row in connection.execute("PRAGMA table_info(scans)") - } - check_ownership = {"target_device", "target_inode"} <= scan_columns - ownership_matches = not check_ownership or _repository_ownership_matches( - connection, - repository, - requested_repository["target_id"], - requested_repository["repository_identity"] if identity_column else None, - ) - live_identity = repository_identity(repository) + identities = RepositoryIdentityCache(connection) + requested = identities.for_path(str(repository)) + check_ownership = {"target_device", "target_inode"} <= identities.scan_columns identity_matches = ( - not identity_column - or requested_repository["repository_identity"] is None - or requested_repository["repository_identity"] == live_identity - ) - ownership_matches = ownership_matches and identity_matches - requested_identity = ( - verified_repository_identity( - connection, - requested_repository["target_id"], - str(repository), - stored_identity=( - requested_repository["repository_identity"] if identity_column else None - ), + requested.resolved_path is not None + and ( + requested.stored_identity is None + or requested.missing + or requested.stored_identity == requested.live_identity ) - if ownership_matches - else None, - None, - None, ) - related_targets = ( - [ - target - for target in connection.execute( - "SELECT id AS target_id, current_path AS target_path, repository_identity " - "FROM security_targets" - if identity_column - else "SELECT id AS target_id, current_path AS target_path FROM security_targets" - ) - if _same_repository( - connection, - target, - requested_repository, - after_identity=requested_identity, - ) - ] - if ownership_matches - else [] - ) - repository_clauses = ( - [ - _owned_scan_clause( - repository, - check_ownership - and not ( - identity_column - and requested_repository["repository_identity"] is not None - and requested_repository["repository_identity"] == live_identity - and ownership_matches - and repository_relative_path(repository) not in (None, ".") - ), - values, - ) - ] - if identity_matches - else ["0"] - ) - for target in related_targets: - target_path = Path(target["target_path"]) - target_identity = target["repository_identity"] if identity_column else None - verified_target = ( - check_ownership - and target_identity is not None - and repository_relative_path(target_path) not in (None, ".") - and _repository_ownership_matches( - connection, target_path, target["target_id"], target_identity - ) + repository_clauses = [ + _owned_scan_clause( + repository, + check_ownership and not requested.relaxed_directory_ownership, + values, ) - repository_clauses.append( - _owned_scan_clause( - target_path, - check_ownership and not verified_target, - values, - target_id=target["target_id"], + ] if identity_matches else ["0"] + requested_identity = requested.verified_identity + if requested_identity is not None: + for target_id, target in identities.targets.items(): + if target_id == requested.target_id or identities.group(target_id) != ( + "repository", requested_identity + ): + continue + candidate = identities.for_row(target) + if candidate.resolved_path is None: + continue + repository_clauses.append( + _owned_scan_clause( + Path(candidate.target_path), + check_ownership and not candidate.relaxed_directory_ownership, + values, + target_id=target_id, + ) ) - ) clauses.append(f"({' OR '.join(repository_clauses)})") if args is not None and args.scan_root: scan_root = str(Path(args.scan_root).expanduser().resolve()) @@ -377,18 +251,6 @@ def _owned_scan_clause( ) -def _repository_ownership_matches( - connection: sqlite3.Connection, - repository: Path, - target_id: str | None, - stored_identity: str | None = None, -) -> bool: - try: - repository.stat() - _require_current_target_owner(connection, target_id or "", str(repository), stored_identity) - except (OSError, SystemExit): - return False - return True def list_unmatched_scan_pairs( @@ -399,61 +261,26 @@ def list_unmatched_scan_pairs( read_coverage: Callable[[sqlite3.Row], dict[str, Any]], ) -> dict[str, Any]: repository = Path(args.repository).expanduser().resolve() - identity_column = supports_repository_identity(connection) - requested_identity_column = ( - ", (SELECT repository_identity FROM security_targets " - "WHERE current_path = ?) AS repository_identity" - if identity_column - else "" - ) - requested_values = ( - (str(repository), str(repository), str(repository)) - if identity_column - else (str(repository), str(repository)) - ) - requested = connection.execute( - f""" - SELECT COALESCE((SELECT id FROM security_targets WHERE current_path = ?), '') AS target_id, - ? AS target_path - {requested_identity_column} - """, - requested_values, - ).fetchone() - if identity_column and requested["target_id"]: - _require_current_target_owner( - connection, - requested["target_id"], - str(repository), - requested["repository_identity"], - ) + identities = RepositoryIdentityCache(connection) + requested = identities.for_path(str(repository)) + if identities.supports_identity and requested.target_id: + requested.require_owner() + requested_row = { + "target_id": requested.target_id, + "target_path": requested.target_path, + "repository_identity": requested.stored_identity, + } scan_query = ( "SELECT scans.*, targets.repository_identity " "FROM scans LEFT JOIN security_targets AS targets ON targets.id = scans.target_id " "WHERE scans.status = 'complete' ORDER BY scans.started_at, scans.id" - if identity_column + if identities.supports_identity else "SELECT * FROM scans WHERE status = 'complete' ORDER BY started_at, id" ) - requested_identity = verified_repository_identity( - connection, - requested["target_id"], - str(repository), - stored_identity=requested["repository_identity"] if identity_column else None, - ) - requested_group = ( - requested_identity, - _repository_origin(repository), - repository_relative_path(repository), - ) selected = [ scan for scan in connection.execute(scan_query) - if str(Path(scan["target_path"]).resolve()) == str(repository) - or _same_repository( - connection, - scan, - requested, - after_identity=requested_group, - ) + if _same_repository(connection, scan, requested_row, identities=identities) ] available = [] diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py index bec4c7b41..345f91595 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py @@ -909,6 +909,9 @@ def normalize_pre_release_migrations(connection: sqlite3.Connection, timestamp: repository_identity_migration is not None and repository_identity_migration["name"] == "persist repository identities" ): + from workbench_target_state import normalize_pre_release_repository_identities + + normalize_pre_release_repository_identities(connection) connection.execute( "UPDATE schema_migrations SET version = 31 WHERE version = 30 AND name = ?", ("persist repository identities",), diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index 198c02d19..324484bff 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -6,10 +6,13 @@ import hashlib import os import sqlite3 +import stat import subprocess import sys +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path +from urllib.parse import urlsplit # Some plugin hosts launch Python with safe-path isolation enabled. sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -77,8 +80,21 @@ def _repository_birth_time_ns(path: str, metadata: os.stat_result) -> int | None return birth_time_ns if birth_time_ns > 0 else None -def repository_identity(target: Path | str) -> str | None: - """Identify matching Git worktree targets without storing remote credentials.""" +@dataclass(frozen=True) +class GitRepositoryIdentity: + value: str + relative_path: str + common_directory: str + device: int | str + inode: int | str + birth_time_ns: int + + +def _identity_digest(material: str) -> str: + return f"repository_sha256_{hashlib.sha256(material.encode()).hexdigest()}" + + +def _repository_identity_details(target: Path | str) -> GitRepositoryIdentity | None: target = Path(target) common_directory = git_output( target, "rev-parse", "--path-format=absolute", "--git-common-dir" @@ -113,7 +129,42 @@ def repository_identity(target: Path | str) -> str | None: f"git-common-dir\0{canonical_directory}\0{device}\0{inode}\0" f"{birth_time_ns}\0{relative}" ) - return f"repository_sha256_{hashlib.sha256(material.encode()).hexdigest()}" + return GitRepositoryIdentity( + _identity_digest(material), relative, canonical_directory, device, inode, birth_time_ns + ) + + +def repository_identity(target: Path | str) -> str | None: + """Identify matching Git worktree targets without storing remote credentials.""" + identity = _repository_identity_details(target) + return identity.value if identity is not None else None + + +def repository_origin(target: Path) -> tuple[str, str] | None: + remote = git_output(target, "remote", "get-url", "origin") + if remote is None: + return None + if "://" in remote: + try: + parsed = urlsplit(remote) + port = parsed.port + except ValueError: + return None + if parsed.scheme not in {"https", "ssh"} or parsed.hostname is None: + return None + if parsed.query or parsed.fragment: + return None + host = parsed.hostname + if port is not None and port != {"https": 443, "ssh": 22}[parsed.scheme]: + host = f"{host}:{port}" + path = parsed.path + else: + authority, separator, path = remote.partition(":") + if not separator or "?" in path or "#" in path: + return None + host = authority.rsplit("@", 1)[-1] + path = path.strip("/").removesuffix(".git") + return (host.lower(), path) if host and path else None def supports_repository_identity(connection: sqlite3.Connection) -> bool: @@ -123,115 +174,281 @@ def supports_repository_identity(connection: sqlite3.Connection) -> bool: ) -def verified_repository_identity( - connection: sqlite3.Connection, - target_id: str, - target_path: str, - *, - stored_identity: str | None = None, -) -> str | None: - target = Path(target_path) - try: - metadata = target.stat() - except OSError: - return None - - if stored_identity is not None: - try: - _require_current_target_owner(connection, target_id, target_path, stored_identity) - except SystemExit: +@dataclass(frozen=True) +class RepositoryTargetState: + target_id: str + target_path: str + stored_identity: str | None + resolved_path: str | None = None + metadata: os.stat_result | None = None + repository: GitRepositoryIdentity | None = None + ownership_matches: bool = False + strict_owner_matches: bool = False + missing: bool = False + + @property + def live_identity(self) -> str | None: + return self.repository.value if self.repository is not None else None + + @property + def verified_identity(self) -> str | None: + if not self.ownership_matches or self.repository is None: return None - identity = repository_identity(target) - return identity if identity == stored_identity else None - - scan_columns = { - row["name"] for row in connection.execute("PRAGMA table_info(scans)") - } - if not {"target_id", "target_path"} <= scan_columns: - return None - if not {"target_device", "target_inode"} <= scan_columns: - historical_scan = connection.execute( - "SELECT 1 FROM scans WHERE target_id = ? OR target_path = ? LIMIT 1", - (target_id, target_path), - ).fetchone() - return None if historical_scan is not None else repository_identity(target) + if self.stored_identity is None: + return self.live_identity if self.strict_owner_matches else None + return self.live_identity if self.live_identity == self.stored_identity else None + + @property + def relaxed_directory_ownership(self) -> bool: + return ( + self.stored_identity is not None + and self.verified_identity == self.stored_identity + and self.repository is not None + and self.repository.relative_path != "." + ) - scans = connection.execute( - """ - SELECT target_device, target_inode - FROM scans - WHERE target_id = ? OR target_path = ? - """, - (target_id, target_path), - ) - if any( - not stored_filesystem_identity_matches(scan["target_device"], metadata.st_dev) - or not stored_filesystem_identity_matches(scan["target_inode"], metadata.st_ino) - for scan in scans - ): - return None - return repository_identity(target) + def require_owner(self) -> None: + if not self.ownership_matches: + raise SystemExit( + f"The repository checkout at {self.target_path} no longer matches its recorded " + "security scan history; refusing to reuse its target." + ) -def _require_current_target_owner( +def _inspect_repository_target( connection: sqlite3.Connection, target_id: str, target_path: str, stored_identity: str | None, -) -> None: + *, + scan_columns: set[str] | None = None, +) -> RepositoryTargetState: target = Path(target_path) + try: + resolved_path = str(target.resolve()) + except (OSError, RuntimeError): + return RepositoryTargetState(target_id, target_path, stored_identity) try: metadata = target.stat() + except (FileNotFoundError, NotADirectoryError): + return RepositoryTargetState( + target_id, target_path, stored_identity, + resolved_path=resolved_path, ownership_matches=True, missing=True, + ) except OSError: - return - scan_columns = { - row["name"] for row in connection.execute("PRAGMA table_info(scans)") - } - if not {"target_id", "target_path", "target_device", "target_inode"} <= scan_columns: - return - scans = connection.execute( - """ - SELECT target_device, target_inode - FROM scans - WHERE target_id = ? OR target_path = ? - """, - (target_id, target_path), - ) + return RepositoryTargetState( + target_id, target_path, stored_identity, resolved_path=resolved_path + ) + repository = _repository_identity_details(target) + if scan_columns is None: + scan_columns = { + row["name"] for row in connection.execute("PRAGMA table_info(scans)") + } historical_scan = False recorded_owner = False malformed_owner = False mismatch = False - for scan in scans: - historical_scan = True - device, inode = scan["target_device"], scan["target_inode"] - if device is None and inode is None: - continue - if device is None or inode is None: - malformed_owner = True - break - recorded_owner = True - if not stored_filesystem_identity_matches( - device, metadata.st_dev - ) or not stored_filesystem_identity_matches(inode, metadata.st_ino): - mismatch = True - verified_repository = ( - stored_identity is not None and repository_identity(target) == stored_identity + strict_owner_matches = False + ownership_matches = True + if {"target_id", "target_path"} <= scan_columns: + if not {"target_device", "target_inode"} <= scan_columns: + historical_scan = connection.execute( + "SELECT 1 FROM scans WHERE target_id = ? OR target_path = ? LIMIT 1", + (target_id, target_path), + ).fetchone() is not None + strict_owner_matches = not historical_scan + else: + strict_owner_matches = True + for scan in connection.execute( + """ + SELECT target_device, target_inode FROM scans + WHERE target_id = ? OR target_path = ? + """, + (target_id, target_path), + ): + historical_scan = True + device, inode = scan["target_device"], scan["target_inode"] + if device is None and inode is None: + strict_owner_matches = False + continue + if device is None or inode is None: + malformed_owner = True + strict_owner_matches = False + continue + recorded_owner = True + if not stored_filesystem_identity_matches( + device, metadata.st_dev + ) or not stored_filesystem_identity_matches(inode, metadata.st_ino): + mismatch = True + strict_owner_matches = False + verified_repository = ( + stored_identity is not None + and repository is not None + and repository.value == stored_identity + ) + ownership_matches = not ( + malformed_owner + or mismatch + and ( + not verified_repository + or repository is None + or repository.relative_path == "." + ) + or stored_identity is not None + and ( + not verified_repository or historical_scan and not recorded_owner + ) + ) + return RepositoryTargetState( + target_id, target_path, stored_identity, resolved_path, metadata, repository, + ownership_matches, strict_owner_matches, ) - if ( - malformed_owner - or mismatch - and ( - not verified_repository - or repository_relative_path(target) in (None, ".") + + +def verified_repository_identity( + connection: sqlite3.Connection, + target_id: str, + target_path: str, + *, + stored_identity: str | None = None, +) -> str | None: + return _inspect_repository_target( + connection, target_id, target_path, stored_identity + ).verified_identity + + +class RepositoryIdentityCache: + """One request's saved identities and verified live aliases.""" + + def __init__(self, connection: sqlite3.Connection) -> None: + self.connection = connection + self.supports_identity = supports_repository_identity(connection) + self.scan_columns = { + row["name"] for row in connection.execute("PRAGMA table_info(scans)") + } + identity_column = "repository_identity" if self.supports_identity else "NULL" + self.targets = { + row["target_id"]: row + for row in connection.execute( + "SELECT id AS target_id, current_path AS target_path, " + f"{identity_column} AS repository_identity FROM security_targets" + ) + } + self.targets_by_path = { + row["target_path"]: row for row in self.targets.values() + } + self._states: dict[tuple[str, str, str | None], RepositoryTargetState] = {} + self._origins: dict[str, tuple[str, str] | None] = {} + + def for_row(self, row: sqlite3.Row | dict) -> RepositoryTargetState: + key = ( + row["target_id"] or "", + row["target_path"], + row["repository_identity"] if "repository_identity" in row.keys() else None, ) - or stored_identity is not None - and ( - not verified_repository or historical_scan and not recorded_owner + if key not in self._states: + self._states[key] = _inspect_repository_target( + self.connection, *key, scan_columns=self.scan_columns + ) + return self._states[key] + + def for_path(self, target_path: str) -> RepositoryTargetState: + row = self.targets_by_path.get(target_path) + return self.for_row( + row if row is not None else { + "target_id": "", "target_path": target_path, "repository_identity": None, + } ) - ): - raise SystemExit( - f"The repository checkout at {target_path} no longer matches its recorded " - "security scan history; refusing to reuse its target." + + def group(self, target_id: str) -> tuple[str, str]: + target = self.targets.get(target_id) + identity = target["repository_identity"] if target is not None else None + if identity is None and target is not None: + identity = self.for_row(target).verified_identity + return ("target", target_id) if identity is None else ("repository", identity) + + def target_ids(self, target_id: str) -> set[str]: + target = self.targets.get(target_id) + if not self.supports_identity or target is None: + return {target_id} + requested = self.for_row(target) + if not requested.ownership_matches: + return set() + group = self.group(target_id) + if group[0] == "target": + return {target_id} + return {candidate for candidate in self.targets if self.group(candidate) == group} + + def origin(self, state: RepositoryTargetState) -> tuple[str, str] | None: + if state.target_path not in self._origins: + self._origins[state.target_path] = repository_origin(Path(state.target_path)) + return self._origins[state.target_path] + + +def _pre_release_repository_identities(identity: GitRepositoryIdentity) -> set[str]: + directory = os.path.normcase(identity.common_directory) + relative = os.path.normcase(os.fspath(Path(identity.relative_path))).replace(os.sep, "/") + prefix = f"git-common-dir\0{directory}\0{identity.device}\0{identity.inode}\0" + identities = {_identity_digest(f"{prefix}{relative}")} + try: + generation = (Path(identity.common_directory) / "description").lstat() + except OSError: + return identities + if stat.S_ISREG(generation.st_mode): + identities.add(_identity_digest( + f"{prefix}git-description\0" + f"{serialize_filesystem_identity(generation.st_dev)}\0" + f"{serialize_filesystem_identity(generation.st_ino)}\0" + f"{generation.st_ctime_ns}\0{relative}" + )) + return identities + + +def _timestamp_ns(value: str) -> int | None: + try: + timestamp = datetime.fromisoformat(value.replace("Z", "+00:00")) + except (AttributeError, ValueError): + return None + if timestamp.tzinfo is None: + return None + delta = timestamp - datetime(1970, 1, 1, tzinfo=timezone.utc) + return (delta.days * 86400 + delta.seconds) * 1_000_000_000 + delta.microseconds * 1000 + + +def normalize_pre_release_repository_identities(connection: sqlite3.Connection) -> None: + """Upgrade only known old hashes whose recorded generation is still present.""" + if not supports_repository_identity(connection): + return + for target in connection.execute( + "SELECT id, current_path, created_at, repository_identity FROM security_targets " + "WHERE repository_identity IS NOT NULL" + ).fetchall(): + stored = target["repository_identity"] + state = _inspect_repository_target( + connection, target["id"], target["current_path"], stored + ) + identity = state.repository + if ( + identity is None + or identity.value == stored + or not state.strict_owner_matches + or stored not in _pre_release_repository_identities(identity) + ): + continue + scans = connection.execute( + "SELECT started_at, created_at FROM scans WHERE target_id = ? OR target_path = ?", + (target["id"], target["current_path"]), + ).fetchall() + timestamps = ( + [_timestamp_ns(scan[column]) for scan in scans for column in ("started_at", "created_at")] + if scans else [_timestamp_ns(target["created_at"])] + ) + if any(value is None for value in timestamps) or identity.birth_time_ns > min(timestamps): + continue + connection.execute( + "UPDATE security_targets SET repository_identity = ? " + "WHERE id = ? AND repository_identity = ?", + (identity.value, target["id"], stored), ) @@ -294,12 +511,14 @@ def ensure_security_target( ).fetchone() if existing is not None: target_id = str(existing["id"]) - if verify_ownership and supports_identity: - _require_current_target_owner( + if supports_identity: + state = _inspect_repository_target( connection, target_id, target_path, existing["repository_identity"] ) + if verify_ownership: + state.require_owner() if supports_identity and existing["repository_identity"] is None: - identity = verified_repository_identity(connection, target_id, target_path) + identity = state.verified_identity if identity is not None: connection.execute( """ diff --git a/sdk/typescript/tests-ts/repository-findings.test.ts b/sdk/typescript/tests-ts/repository-findings.test.ts index 9a82060a0..05fce8b3a 100644 --- a/sdk/typescript/tests-ts/repository-findings.test.ts +++ b/sdk/typescript/tests-ts/repository-findings.test.ts @@ -226,6 +226,15 @@ result = { "repositories": indexes.list_repositories(connection), "openRepositories": indexes.list_repositories(connection, repository_arguments), } +connection.execute( + "UPDATE finding_occurrences SET created_at = ? WHERE id = ?", + ("2026-02-05T00:00:00Z", "same-id-primary"), +) +result["overlappingCompletions"] = findings("primary", None) +connection.execute( + "UPDATE finding_occurrences SET created_at = ? WHERE id = ?", + ("2026-02-03T00:00:00Z", "same-id-primary"), +) connection.execute( "INSERT INTO finding_triage VALUES (?, ?, ?, ?)", ("same-id-primary", "closed", "2026-02-05T00:00:00Z", "false_positive"), @@ -277,6 +286,15 @@ print(json.dumps(result)) >; const primary = result["primary"]!.findings; expect(result["linked"]!.findings).toEqual(primary); + expect( + result["overlappingCompletions"]!.findings.find( + (finding) => finding["findingId"] === "same-id", + ), + ).toMatchObject({ + scanId: "primary-open", + confirmedInLatestScan: true, + knownScanIds: ["primary-open", "linked-latest"], + }); expect(primary.map((finding) => finding["findingId"])).toEqual([ "linked-only", "same-id", diff --git a/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts b/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts index cfc527422..c75ae7213 100644 --- a/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts +++ b/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts @@ -58,6 +58,7 @@ const caseSensitiveWindowsHistoryProbe = [ "from types import SimpleNamespace", "sys.path.insert(0, sys.argv[1])", "import workbench_scan_history as history", + "import workbench_target_state as target_state", "class WindowsPath(PureWindowsPath):", " def expanduser(self):", " return self", @@ -82,25 +83,25 @@ const caseSensitiveWindowsHistoryProbe = [ "same_path = target('same-path', 'D:/Repository/Service', None)", "targets = [upper_root, lower_root, upper_scope, lower_scope, linked_scope, clone_scope]", "identities = {entry['target_path']: entry['repository_identity'] for entry in targets}", - "history.repository_identity = lambda path: identities.get(str(path))", - "history.verified_repository_identity = lambda connection, target_id, path, **kwargs: identities.get(str(path))", - "history.repository_relative_path = lambda path: WindowsPath(path).name", - "history._repository_origin = lambda path: ('example.test', 'synthetic/repository')", - "root_group = ('root-upper', None, None)", - "scope_group = ('scope-upper', None, None)", + "def inspect(database, target_id, path, stored, **kwargs):", + " identity = identities.get(path)", + " repository = target_state.GitRepositoryIdentity(identity, WindowsPath(path).name, 'synthetic', 1, 2, 3) if identity else None", + " return target_state.RepositoryTargetState(target_id, path, stored, resolved_path=str(WindowsPath(path)), repository=repository, ownership_matches=True, strict_owner_matches=True)", + "target_state._inspect_repository_target = inspect", + "target_state.repository_origin = lambda path: ('example.test', 'synthetic/repository')", + "for entry in targets:", + " connection.execute('INSERT INTO security_targets VALUES (?, ?, ?)', (entry['target_id'], entry['target_path'], entry['repository_identity']))", + "cache = history.RepositoryIdentityCache(connection)", "checks = {", " 'nativeWindowsScopeEquality': WindowsPath(upper_scope['target_path']) == WindowsPath(lower_scope['target_path']),", " 'nativeWindowsRootEquality': WindowsPath(upper_root['target_path']) == WindowsPath(lower_root['target_path']),", - " 'caseSensitiveScopesMatch': history._same_repository(connection, lower_scope, upper_scope, after_identity=scope_group),", - " 'caseSensitiveRootsMatch': history._same_repository(connection, lower_root, upper_root, after_identity=root_group),", - " 'linkedWorktreeMatches': history._same_repository(connection, linked_scope, upper_scope, after_identity=scope_group),", - " 'sameOriginCloneMatches': history._same_repository(connection, clone_scope, upper_scope, after_identity=scope_group),", - " 'exactResolvedPathMatches': history._same_repository(connection, same_path, upper_scope, after_identity=scope_group),", + " 'caseSensitiveScopesMatch': cache.group(lower_scope['target_id']) == cache.group(upper_scope['target_id']),", + " 'caseSensitiveRootsMatch': cache.group(lower_root['target_id']) == cache.group(upper_root['target_id']),", + " 'linkedWorktreeMatches': cache.group(linked_scope['target_id']) == cache.group(upper_scope['target_id']),", + " 'sameOriginCloneMatches': cache.group(clone_scope['target_id']) == cache.group(upper_scope['target_id']),", + " 'exactResolvedPathMatches': history._same_repository(connection, same_path, upper_scope, identities=cache),", "}", - "for entry in (upper_scope, lower_scope):", - " connection.execute('INSERT INTO security_targets VALUES (?, ?, ?)', (entry['target_id'], entry['target_path'], entry['repository_identity']))", "connection.execute('INSERT INTO scans VALUES (?, ?, ?, ?, ?)', ('lower-scan', lower_scope['target_id'], lower_scope['target_path'], 'complete', '2026-08-15T00:00:00Z'))", - "history._require_current_target_owner = lambda *args: None", "history._same_repository = lambda *args, **kwargs: False", "selected = history.list_unmatched_scan_pairs(connection, SimpleNamespace(repository=upper_scope['target_path'], force=False), backfill_finding_details=lambda *args: None, read_coverage=lambda scan: {})", "checks['caseSensitiveUnmatchedScanCount'] = selected['scanCount']", diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts new file mode 100644 index 000000000..0257820d2 --- /dev/null +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -0,0 +1,371 @@ +import { spawnSync } from "node:child_process"; +import { join } from "node:path"; +import { expect, test } from "bun:test"; +import { PLUGIN_ROOT } from "./plugin-root.js"; + +const probe = String.raw` +import argparse +import hashlib +import json +import os +import sqlite3 +import stat +import sys +from collections import Counter +from contextlib import ExitStack +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +sys.path.insert(0, sys.argv[1]) +import workbench_scan_history as history +import workbench_native_indexes as indexes +import workbench_target_state as state +from workbench_feedback import get_scan_feedback +from workbench_schema import MIGRATIONS, apply_migrations + +scenario = sys.argv[2] +timestamp = "2026-08-01T00:00:00Z" +root = Path.cwd() / "synthetic-identity-fixture" +connection = sqlite3.connect(":memory:") +connection.row_factory = sqlite3.Row +connection.execute("PRAGMA foreign_keys = ON") +identity_migration = next(item for item in MIGRATIONS if item[0] == 31) +initial = ( + (*tuple(item for item in MIGRATIONS if item[0] <= 28), (30, *identity_migration[1:])) + if scenario == "migration" else MIGRATIONS +) +apply_migrations(connection, initial, lambda: timestamp, lambda database: None) +paths = {} +details = {} +metadata = {} +missing = set() +resolution_errors = {} +probes = Counter() +origins = Counter() +description = SimpleNamespace( + st_mode=stat.S_IFREG, st_dev=44, st_ino=55, st_ctime_ns=66 +) + + +def add_target(name, stored, live=None, relative=".", birth=1_000_000_000): + path = str(root / name) + paths[name] = path + metadata[path] = SimpleNamespace(st_mode=stat.S_IFDIR, st_dev=7, st_ino=100 + len(paths)) + details[path] = state.GitRepositoryIdentity( + live or stored or "repository-current", relative, str(root / "common"), 11, 22, birth + ) + connection.execute( + "INSERT INTO security_targets " + "(id, current_path, display_name, created_at, updated_at, repository_identity) " + "VALUES (?, ?, ?, ?, ?, ?)", + (name, path, name, timestamp, timestamp, stored), + ) + return details[path] + + +def add_scan(scan_id, target, owner="current", started=timestamp): + path = paths[target] + device, inode = metadata[path].st_dev, metadata[path].st_ino + if owner == "missing": + device, inode = None, None + elif owner == "mismatch": + inode += 1 + connection.execute( + "INSERT INTO workspaces (id, target_path, target_id, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + ("workspace-" + scan_id, path, target, timestamp, timestamp), + ) + connection.execute( + "INSERT INTO scans (id, workspace_id, target_path, target_id, target_device, " + "target_inode, target_revision, scope, mode, scan_dir, status, phase, " + "started_at, completed_at, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (scan_id, "workspace-" + scan_id, path, target, device, inode, "synthetic", + ".", "standard", str(root / "scans" / scan_id), "complete", "reporting", + started, timestamp, timestamp, timestamp), + ) + connection.execute( + "INSERT INTO scan_progress (scan_id, updated_at) VALUES (?, ?)", + (scan_id, timestamp), + ) + + +def add_finding(scan_id, finding_id, closed=False): + occurrence = scan_id + ":" + finding_id + connection.execute( + "INSERT OR IGNORE INTO findings " + "(id, fingerprint, rule_id, identity_anchor, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?)", + (finding_id, "fingerprint-" + finding_id, "synthetic-rule", finding_id, timestamp, timestamp), + ) + connection.execute( + "INSERT INTO finding_occurrences " + "(id, finding_id, scan_id, title, summary, severity, confidence, remediation, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + (occurrence, finding_id, scan_id, finding_id, "Synthetic summary", "high", + "high", "Synthetic remediation", timestamp), + ) + connection.execute( + "INSERT INTO finding_locations " + "(occurrence_id, relative_path, start_line, end_line, role, sort_order) " + "VALUES (?, ?, ?, ?, ?, ?)", + (occurrence, "src/example.py", 1, 1, "root_control", 0), + ) + if closed: + connection.execute( + "INSERT INTO finding_triage VALUES (?, ?, ?, ?, ?)", + (occurrence, "closed", "false_positive", "Synthetic review", timestamp), + ) + return occurrence + + +original_resolve, original_stat, original_lstat = Path.resolve, Path.stat, Path.lstat + + +def resolve_path(path, *args, **kwargs): + value = str(path) + if value in resolution_errors: + raise resolution_errors[value]("Synthetic path resolution failure") + return path if value in metadata else original_resolve(path, *args, **kwargs) + + +def stat_path(path, *args, **kwargs): + value = str(path) + if value in missing: + raise FileNotFoundError(value) + return metadata[value] if value in metadata else original_stat(path, *args, **kwargs) + + +def lstat_path(path, *args, **kwargs): + if path.name == "description" and path.parent == root / "common": + return description + return original_lstat(path, *args, **kwargs) + + +def identity_details(path): + value = str(path) + probes[value] += 1 + return details.get(value) + + +def origin(path): + value = str(path) + origins[value] += 1 + identity = details.get(value) + return ("example.test", identity.value) if identity is not None else None + + +def listed(target): + args = argparse.Namespace( + repository=paths[target], scan_root=None, target_id=None, mode=None, + status=None, query=None, limit=None, offset=0, + ) + return sorted(row["scanId"] for row in history.list_scans(connection, args)["scans"]) + + +def findings(target): + return indexes.list_global_findings( + connection, + argparse.Namespace(target_id=target, limit=50, offset=0, query=None, severity=None, status=None), + )["findings"] + + +def legacy_hash(identity, with_description=False): + directory = os.path.normcase(identity.common_directory) + relative = os.path.normcase(os.fspath(Path(identity.relative_path))).replace(os.sep, "/") + material = f"git-common-dir\0{directory}\0{identity.device}\0{identity.inode}\0" + if with_description: + material += f"git-description\0{description.st_dev}\0{description.st_ino}\0{description.st_ctime_ns}\0" + return "repository_sha256_" + hashlib.sha256((material + relative).encode()).hexdigest() + + +with ExitStack() as stack: + stack.enter_context(patch.object(Path, "resolve", resolve_path)) + stack.enter_context(patch.object(Path, "stat", stat_path)) + stack.enter_context(patch.object(Path, "lstat", lstat_path)) + stack.enter_context(patch.object(state, "_repository_identity_details", identity_details)) + stack.enter_context(patch.object(state, "repository_origin", origin)) + if scenario == "cache": + for name, stored, live in [ + ("requested", "repository-current", None), + ("persisted", "repository-current", None), + ("legacy", None, "repository-current"), + ("unverified", None, "repository-current"), + ("unrelated", "repository-other", None), + ("unresolvable-runtime", None, "repository-current"), + ("unresolvable-os", None, "repository-current"), + ("removed", "repository-current", None), + ("changed", "repository-previous", "repository-current"), + ]: + add_target(name, stored, live) + for name in paths: + add_scan(name + "-scan", name, "missing" if name == "unverified" else "current") + for number in range(5): + add_scan("unrelated-extra-" + str(number), "unrelated") + add_scan("legacy-second", "legacy") + missing.add(paths["removed"]) + resolution_errors[paths["unresolvable-runtime"]] = RuntimeError + resolution_errors[paths["unresolvable-os"]] = OSError + before = add_finding("persisted-scan", "before-review") + after = add_finding("legacy-scan", "after-review", closed=True) + add_finding("legacy-second", "legacy-open") + connection.execute( + "INSERT INTO scan_comparisons VALUES (?, ?, ?, ?, ?)", + ("persisted-scan", "legacy-scan", "{}", timestamp, timestamp), + ) + connection.execute( + "INSERT INTO scan_comparison_matches VALUES (?, ?, ?, ?, ?)", + ("persisted-scan", "legacy-scan", before, after, "Synthetic match"), + ) + scans = listed("requested") + listing_probes = dict(probes) + probes.clear() + origins.clear() + matching = history.list_unmatched_scan_pairs( + connection, argparse.Namespace(repository=paths["requested"], force=False), + backfill_finding_details=lambda database, scan: None, + read_coverage=lambda scan: {}, + ) + matching_probes = dict(probes) + matching_origins = dict(origins) + probes.clear() + indexed = findings("requested") + indexing_probes = dict(probes) + probes.clear() + feedback = get_scan_feedback( + connection, connection.execute("SELECT * FROM scans WHERE id = 'requested-scan'").fetchone() + ) + feedback_probes = dict(probes) + print(json.dumps({ + "scans": scans, + "removedExact": listed("removed"), + "matchingCount": matching["scanCount"], + "aliases": sorted(indexes.repository_target_ids(connection, "requested")), + "findings": indexed, + "feedback": [row["findingId"] for row in feedback["falsePositives"]], + "listingRequestedProbes": listing_probes.get(paths["requested"], 0), + "matchingRequestedProbes": matching_probes.get(paths["requested"], 0), + "matchingUnrelatedProbes": matching_probes.get(paths["unrelated"], 0), + "matchingUnrelatedOrigins": matching_origins.get(paths["unrelated"], 0), + "indexingMaxProbes": max(indexing_probes.values(), default=0), + "feedbackMaxProbes": max(feedback_probes.values(), default=0), + "legacyStored": connection.execute( + "SELECT repository_identity FROM security_targets WHERE id = 'legacy'" + ).fetchone()[0], + "changedStored": connection.execute( + "SELECT repository_identity FROM security_targets WHERE id = 'changed'" + ).fetchone()[0], + })) + else: + stack.enter_context(patch.object(os.path, "normcase", lambda value: os.fspath(value).lower())) + originals = {} + expected = {} + for name in [ + "old-basic", "old-description", "current", "unknown", "newer-generation", + "unverified", "changed-owner", "unavailable", "invalid-time", "no-scans", + "scope-upper", "scope-lower", + ]: + relative = "Service" if name == "scope-upper" else "service" if name == "scope-lower" else "." + birth = state._timestamp_ns("2026-08-02T00:00:00Z") if name == "newer-generation" else 1_000_000_000 + identity = add_target(name, None, "current-" + name, relative, birth) + stored = ( + identity.value if name == "current" else + "unknown-identity" if name == "unknown" else + legacy_hash(identity, name == "old-description") + ) + originals[name] = stored + connection.execute( + "UPDATE security_targets SET repository_identity = ? WHERE id = ?", (stored, name) + ) + if name != "no-scans": + owner = "missing" if name == "unverified" else "mismatch" if name == "changed-owner" else "current" + add_scan(name + "-scan", name, owner, "invalid" if name == "invalid-time" else timestamp) + if name == "unavailable": + missing.add(paths[name]) + expected[name] = identity.value if name in { + "old-basic", "old-description", "current", "no-scans", "scope-upper", "scope-lower" + } else stored + apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) + first = { + row["id"]: row["repository_identity"] + for row in connection.execute("SELECT id, repository_identity FROM security_targets") + } + apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) + second = { + row["id"]: row["repository_identity"] + for row in connection.execute("SELECT id, repository_identity FROM security_targets") + } + print(json.dumps({ + "identities": first, + "expected": expected, + "idempotent": first == second, + "foldedLegacyScopesEqual": originals["scope-upper"] == originals["scope-lower"], + "currentScopesDistinct": first["scope-upper"] != first["scope-lower"], + "targetCount": len(first), + "migrations": [row["version"] for row in connection.execute("SELECT version FROM schema_migrations ORDER BY version")], + })) +`; + +function run(scenario: string): Record { + const python = (Bun.which("python3") ?? Bun.which("python"))!; + const execution = spawnSync( + python, + ["-I", "-B", "-c", probe, join(PLUGIN_ROOT, "scripts"), scenario], + { encoding: "utf8", timeout: 10_000 }, + ); + expect(execution.status, execution.stderr).toBe(0); + return JSON.parse(execution.stdout) as Record; +} + +test("reuses verified legacy aliases and probes each saved target once per request", () => { + const result = run("cache"); + + expect(result["scans"]).toEqual([ + "legacy-scan", + "legacy-second", + "persisted-scan", + "removed-scan", + "requested-scan", + ]); + expect(result["removedExact"]).toEqual(["removed-scan"]); + expect(result["matchingCount"]).toBe(5); + expect(result["aliases"]).toEqual([ + "legacy", + "persisted", + "removed", + "requested", + ]); + const findings = result["findings"] as Array>; + expect( + findings.find((finding) => finding["status"] === "closed"), + ).toMatchObject({ + occurrenceCount: 2, + matchedFindingIds: ["after-review", "before-review"], + }); + expect( + findings.find((finding) => finding["findingId"] === "legacy-open"), + ).toBeDefined(); + expect(result["feedback"]).toEqual(["after-review"]); + expect(result["listingRequestedProbes"]).toBe(1); + expect(result["matchingRequestedProbes"]).toBe(1); + expect(result["matchingUnrelatedProbes"]).toBe(1); + expect(result["matchingUnrelatedOrigins"]).toBe(1); + expect(result["indexingMaxProbes"]).toBe(1); + expect(result["feedbackMaxProbes"]).toBe(1); + expect(result["legacyStored"]).toBeNull(); + expect(result["changedStored"]).toBe("repository-previous"); +}); + +test("upgrades only independently verified pre-release repository hashes", () => { + const result = run("migration"); + + expect(result["identities"]).toEqual(result["expected"]); + expect(result["idempotent"]).toBe(true); + expect(result["foldedLegacyScopesEqual"]).toBe(true); + expect(result["currentScopesDistinct"]).toBe(true); + expect(result["targetCount"]).toBe(12); + expect(result["migrations"]).toEqual( + Array.from({ length: 31 }, (_, index) => index + 1), + ); +}); diff --git a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts index 186b08f48..d6e15ab4c 100644 --- a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts +++ b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts @@ -73,6 +73,7 @@ import shutil import sqlite3 import subprocess import sys +from dataclasses import replace from pathlib import Path from types import SimpleNamespace from unittest.mock import patch @@ -85,6 +86,7 @@ from workbench_native_indexes import repository_target_ids from workbench_schema import MIGRATIONS, apply_migrations from workbench_target_state import ( _repository_birth_time_ns, + _repository_identity_details, backfill_repository_identities, backfill_security_targets, ensure_security_target, @@ -573,7 +575,8 @@ elif scenario == "candidate-generation": candidate_identity = repository_identity(clone) def live_identity(target): - return requested_identity if Path(target) == clone else repository_identity(target) + details = _repository_identity_details(target) + return replace(details, value=requested_identity) if Path(target) == clone else details def automatic_history(): matching = history.list_unmatched_scan_pairs( @@ -595,10 +598,7 @@ elif scenario == "candidate-generation": "matchingScanIds": sorted(matching_ids), } - with ( - patch("workbench_target_state.repository_identity", side_effect=live_identity), - patch("workbench_scan_history.repository_identity", side_effect=live_identity), - ): + with patch("workbench_target_state._repository_identity_details", side_effect=live_identity): persisted = automatic_history() connection.execute( "UPDATE security_targets SET repository_identity = NULL WHERE id = ?", From 284acf78608fe6d03fa48efbe859baf72989a145 Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sun, 16 Aug 2026 01:39:37 -0700 Subject: [PATCH 08/21] fix(workbench): preserve scan history ownership and replay --- .../_bundled_plugin/scripts/workbench_cli.py | 1 + .../_bundled_plugin/scripts/workbench_db.py | 2 +- .../scripts/workbench_feedback.py | 5 +- .../scripts/workbench_native_indexes.py | 33 ++-- .../scripts/workbench_scan_history.py | 17 +- .../scripts/workbench_target_state.py | 58 +++++-- sdk/typescript/src/cli.ts | 5 +- sdk/typescript/src/scan-comparison.ts | 2 + sdk/typescript/tests-ts/cli-workbench.test.ts | 9 +- .../tests-ts/scan-comparison.test.ts | 62 ++++++++ .../tests-ts/workbench-identity-cache.test.ts | 149 ++++++++++++++++-- .../workbench-repository-identity.test.ts | 3 +- .../tests-ts/workbench-scan-history.test.ts | 30 +++- 13 files changed, 333 insertions(+), 43 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py b/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py index 77e9d56ac..13276e7e5 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py @@ -137,6 +137,7 @@ def parse_args(description: str) -> argparse.Namespace: list_unmatched_scan_pairs = subparsers.add_parser("list-unmatched-scan-pairs") list_unmatched_scan_pairs.add_argument("--repository", required=True) + list_unmatched_scan_pairs.add_argument("--after-scan-id") list_unmatched_scan_pairs.add_argument("--force", action="store_true") register_cli_scan = subparsers.add_parser("register-cli-scan") diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index 5584d3a13..ab540df25 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -1651,7 +1651,7 @@ def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace) target_id = ensure_security_target(connection, str(repository)) if parent_scan_id is not None: parent = require_scan(connection, parent_scan_id) - if parent["target_id"] != target_id: + if parent["target_id"] not in native_indexes.repository_target_ids(connection, target_id): raise SystemExit("A rerun must belong to the same repository as its parent scan.") connection.execute( diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py b/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py index e92ec96f6..0b1b525a1 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py @@ -29,8 +29,9 @@ def get_scan_feedback(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict indexed_findings = { finding_id: finding - for finding in _indexed_findings(connection, identities=identities) - if finding["target_id"] in target_ids + for finding in _indexed_findings( + connection, identities=identities, target_ids=set(target_ids) + ) for finding_id in finding["matched_finding_ids"] } target_placeholders = ", ".join("?" for _ in target_ids) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py index 0f16dbba7..203b17a21 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py @@ -38,9 +38,8 @@ def list_global_findings( ) findings = ( row - for row in _indexed_findings(connection, identities=identities) - if (target_ids is None or row["target_id"] in target_ids) - and (args.severity is None or row["severity"] == args.severity) + for row in _indexed_findings(connection, identities=identities, target_ids=target_ids) + if (args.severity is None or row["severity"] == args.severity) and (args.status is None or row["status"] == args.status) and ( not query @@ -92,8 +91,19 @@ def _indexed_findings( connection: sqlite3.Connection, *, identities: RepositoryIdentityCache | None = None, + target_ids: set[str] | None = None, ) -> Iterator[dict[str, Any]]: + if target_ids is not None and not target_ids: + return identities = identities or RepositoryIdentityCache(connection) + target_values = tuple(sorted(target_ids)) if target_ids is not None else () + placeholders = ", ".join("?" for _ in target_values) + target_filter = f"scans.target_id IN ({placeholders})" if target_ids is not None else "1" + match_filter = ( + f"before_scans.target_id IN ({placeholders}) " + f"AND after_scans.target_id IN ({placeholders})" + if target_ids is not None else "1" + ) parents: dict[ tuple[tuple[str, str], str], tuple[tuple[str, str], str] ] = {} @@ -104,7 +114,7 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: return identity for match in connection.execute( - """ + f""" SELECT before_scans.target_id AS before_target_id, after_scans.target_id AS after_target_id, before.finding_id AS before_finding_id, @@ -116,7 +126,9 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: JOIN finding_occurrences AS after ON after.id = matches.after_occurrence_id JOIN scans AS after_scans ON after_scans.id = after.scan_id JOIN security_targets AS after_targets ON after_targets.id = after_scans.target_id - """ + WHERE {match_filter} + """, + (*target_values, *target_values), ): before_repository = identities.group(match["before_target_id"]) after_repository = identities.group(match["after_target_id"]) @@ -140,19 +152,20 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: latest_scan_by_repository = { identities.group(row["target_id"]): row["id"] for row in connection.execute( - """ + f""" SELECT scans.target_id, scans.id FROM scans JOIN security_targets AS targets ON targets.id = scans.target_id - WHERE scans.status = 'complete' + WHERE scans.status = 'complete' AND {target_filter} ORDER BY scans.started_at, scans.id - """ + """, + target_values, ) } grouped: dict[tuple[tuple[str, str], str], list[sqlite3.Row]] = {} for row in connection.execute( - """ + f""" SELECT occurrences.id AS occurrence_id, occurrences.finding_id, @@ -182,7 +195,9 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: JOIN scans ON scans.id = occurrences.scan_id JOIN security_targets AS targets ON targets.id = scans.target_id LEFT JOIN finding_triage AS triage ON triage.occurrence_id = occurrences.id + WHERE {target_filter} """, + target_values, ): grouped.setdefault( group( diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py index bc220b678..83310db52 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py @@ -292,19 +292,28 @@ def list_unmatched_scan_pairs( available.append(scan) saved_pairs = { - (row["before_scan_id"], row["after_scan_id"]) + frozenset((row["before_scan_id"], row["after_scan_id"])) for row in connection.execute("SELECT before_scan_id, after_scan_id FROM scan_comparisons") } + focus_scan_id = getattr(args, "after_scan_id", None) + if focus_scan_id is not None and not any(scan["id"] == focus_scan_id for scan in selected): + raise SystemExit("The scan to match is not a completed scan in this repository.") batches = [] skipped = 0 matching_findings: dict[str, list[dict[str, Any]]] = {} for index, after in enumerate(available): + if focus_scan_id is not None and after["id"] != focus_scan_id: + continue + candidates = ( + available[:index] if focus_scan_id is None + else [before for before in available if before["id"] != focus_scan_id] + ) previous = [ before - for before in available[:index] - if args.force or (before["id"], after["id"]) not in saved_pairs + for before in candidates + if args.force or frozenset((before["id"], after["id"])) not in saved_pairs ] - skipped += index - len(previous) + skipped += len(candidates) - len(previous) if not previous: continue for scan in (*previous, after): diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index 324484bff..9540e1920 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -184,6 +184,7 @@ class RepositoryTargetState: repository: GitRepositoryIdentity | None = None ownership_matches: bool = False strict_owner_matches: bool = False + generation_matches_history: bool = False missing: bool = False @property @@ -195,7 +196,11 @@ def verified_identity(self) -> str | None: if not self.ownership_matches or self.repository is None: return None if self.stored_identity is None: - return self.live_identity if self.strict_owner_matches else None + return ( + self.live_identity + if self.strict_owner_matches and self.generation_matches_history + else None + ) return self.live_identity if self.live_identity == self.stored_identity else None @property @@ -249,6 +254,7 @@ def _inspect_repository_target( malformed_owner = False mismatch = False strict_owner_matches = False + generation_matches_history = False ownership_matches = True if {"target_id", "target_path"} <= scan_columns: if not {"target_device", "target_inode"} <= scan_columns: @@ -257,15 +263,23 @@ def _inspect_repository_target( (target_id, target_path), ).fetchone() is not None strict_owner_matches = not historical_scan + generation_matches_history = repository is not None and not historical_scan else: strict_owner_matches = True - for scan in connection.execute( - """ - SELECT target_device, target_inode FROM scans + timestamps = ", started_at, created_at" if { + "started_at", "created_at" + } <= scan_columns else "" + scans = connection.execute( + f""" + SELECT target_device, target_inode{timestamps} FROM scans WHERE target_id = ? OR target_path = ? """, (target_id, target_path), - ): + ).fetchall() + generation_matches_history = ( + repository is not None and _repository_predates_history(repository, scans) + ) + for scan in scans: historical_scan = True device, inode = scan["target_device"], scan["target_inode"] if device is None and inode is None: @@ -298,10 +312,14 @@ def _inspect_repository_target( and ( not verified_repository or historical_scan and not recorded_owner ) + or stored_identity is None + and repository is not None + and historical_scan + and not generation_matches_history ) return RepositoryTargetState( target_id, target_path, stored_identity, resolved_path, metadata, repository, - ownership_matches, strict_owner_matches, + ownership_matches, strict_owner_matches, generation_matches_history, ) @@ -415,6 +433,26 @@ def _timestamp_ns(value: str) -> int | None: return (delta.days * 86400 + delta.seconds) * 1_000_000_000 + delta.microseconds * 1000 +def _repository_predates_history( + identity: GitRepositoryIdentity, + scans: list[sqlite3.Row], + *, + empty_timestamp: str | None = None, +) -> bool: + if not scans and empty_timestamp is None: + return True + if any(not {"started_at", "created_at"} <= set(scan.keys()) for scan in scans): + return False + timestamps = ( + [_timestamp_ns(scan[column]) for scan in scans for column in ("started_at", "created_at")] + if scans else [_timestamp_ns(empty_timestamp)] + ) + return ( + all(value is not None for value in timestamps) + and identity.birth_time_ns <= min(timestamps) + ) + + def normalize_pre_release_repository_identities(connection: sqlite3.Connection) -> None: """Upgrade only known old hashes whose recorded generation is still present.""" if not supports_repository_identity(connection): @@ -439,11 +477,9 @@ def normalize_pre_release_repository_identities(connection: sqlite3.Connection) "SELECT started_at, created_at FROM scans WHERE target_id = ? OR target_path = ?", (target["id"], target["current_path"]), ).fetchall() - timestamps = ( - [_timestamp_ns(scan[column]) for scan in scans for column in ("started_at", "created_at")] - if scans else [_timestamp_ns(target["created_at"])] - ) - if any(value is None for value in timestamps) or identity.birth_time_ns > min(timestamps): + if not _repository_predates_history( + identity, scans, empty_timestamp=target["created_at"] + ): continue connection.execute( "UPDATE security_targets SET repository_identity = ? " diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index f20876ed0..b6b1437ec 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -1418,6 +1418,9 @@ export async function main( scanId, ]); scanArguments = scanArgumentsFromRecipe(recipe, scanId); + if (args.scanId === undefined) { + scanArguments.repository = dependencies.currentDirectory(); + } scanArguments.verbose = options.verbose; } catch (error) { const message = errorMessage(error); @@ -4284,7 +4287,7 @@ function printScanSummary( repositoryFindings?.filter((finding) => finding.confirmedInLatestScan) .length ?? 0; const findingSummary = repositoryFindings?.length - ? `${confirmedCount} confirmed this scan; ${findingCount - confirmedCount} previously found; ${severitySummary}` + ? `${confirmedCount} confirmed in latest repository scan; ${findingCount - confirmedCount} previously found; ${severitySummary}` : severitySummary; const findingColor = findingCount === 0 diff --git a/sdk/typescript/src/scan-comparison.ts b/sdk/typescript/src/scan-comparison.ts index 520f38e27..f1c03e04f 100644 --- a/sdk/typescript/src/scan-comparison.ts +++ b/sdk/typescript/src/scan-comparison.ts @@ -177,6 +177,8 @@ export async function matchCompletedScan( "list-unmatched-scan-pairs", "--repository", options.repository, + "--after-scan-id", + options.scanId, ])) as { batches?: { afterScanId: string; diff --git a/sdk/typescript/tests-ts/cli-workbench.test.ts b/sdk/typescript/tests-ts/cli-workbench.test.ts index 0a2c3ff5c..d56c34437 100644 --- a/sdk/typescript/tests-ts/cli-workbench.test.ts +++ b/sdk/typescript/tests-ts/cli-workbench.test.ts @@ -72,7 +72,7 @@ describe("CLI workbench", () => { ).toBe(0); expect(stderr.text()).toContain( confirmed.length - ? "FINDINGS 2 (1 confirmed this scan; 1 previously found; 2 high)" + ? "FINDINGS 2 (1 confirmed in latest repository scan; 1 previously found; 2 high)" : "FINDINGS 0\n", ); } @@ -843,6 +843,7 @@ describe("CLI workbench", () => { test("reruns the latest completed scan by default", async () => { let parentScanId: unknown; + let rerunRepository: string | undefined; expect( await main( @@ -850,7 +851,8 @@ describe("CLI workbench", () => { capture().stream, capture().stream, dependencies({ - onTurn: (_repository, options) => { + onTurn: (repository, options) => { + rerunRepository = repository; parentScanId = (options as { parentScanId?: string }).parentScanId; }, onWorkbench: (args): JsonObject => @@ -858,7 +860,7 @@ describe("CLI workbench", () => { ? { scans: [{ scanId: "latest-scan" }] } : { recipe: { - repository: "/current/repository", + repository: "/removed/linked-worktree", target: { kind: "repository", paths: [] }, mode: "standard", config: {}, @@ -868,6 +870,7 @@ describe("CLI workbench", () => { ), ).toBe(0); expect(parentScanId).toBe("latest-scan"); + expect(rerunRepository).toBe("/current/repository"); }); test("reruns canonical recipes with exact config, policy, plugin, and lineage", async () => { diff --git a/sdk/typescript/tests-ts/scan-comparison.test.ts b/sdk/typescript/tests-ts/scan-comparison.test.ts index 3300242ae..d25ca1b96 100644 --- a/sdk/typescript/tests-ts/scan-comparison.test.ts +++ b/sdk/typescript/tests-ts/scan-comparison.test.ts @@ -320,6 +320,13 @@ describe("semantic scan comparison", () => { }, }); expect(input).toEqual({ before: [open, dismissed], after: [after] }); + expect(commands[0]).toEqual([ + "list-unmatched-scan-pairs", + "--repository", + "/repository", + "--after-scan-id", + "current", + ]); expect(commands.map(([command]) => command)).toEqual([ "list-unmatched-scan-pairs", "save-scan-comparison", @@ -331,6 +338,61 @@ describe("semantic scan comparison", () => { expect(saved.uncertain).toEqual([]); }); + test("matches a later-started alias when the earlier scan completes last", async () => { + const later = { + findingId: "later-finding", + occurrenceId: "later-occurrence", + }; + const earlier = { + findingId: "earlier-finding", + occurrenceId: "earlier-occurrence", + }; + const commands: (readonly string[])[] = []; + await matchCompletedScan({ + scanId: "earlier-scan", + repository: "/repository", + previousFindings: [later], + falsePositives: [], + findings: [earlier], + async workbench(args) { + commands.push(args); + return args[0] === "list-unmatched-scan-pairs" + ? { + batches: [ + { + afterScanId: "earlier-scan", + afterFindings: [earlier], + beforeScans: [{ scanId: "later-scan", findings: [later] }], + }, + ], + } + : {}; + }, + async matchFindings(input) { + expect(input).toEqual({ before: [later], after: [earlier] }); + return { + matches: [ + { + beforeOccurrenceIds: ["later-occurrence"], + afterOccurrenceIds: ["earlier-occurrence"], + confidence: "high", + reason: "Same synthetic finding.", + }, + ], + uncertain: [], + }; + }, + }); + expect(commands[0]?.slice(-2)).toEqual(["--after-scan-id", "earlier-scan"]); + expect(commands[1]?.slice(0, 5)).toEqual([ + "save-scan-comparison", + "--before-scan-id", + "later-scan", + "--after-scan-id", + "earlier-scan", + ]); + }); + test.each([ ["no history", false, false, false, 0, false], ["a stable identity", true, false, true, 2, false], diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts index 0257820d2..859ddaa93 100644 --- a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -21,6 +21,8 @@ sys.path.insert(0, sys.argv[1]) import workbench_scan_history as history import workbench_native_indexes as indexes import workbench_target_state as state +import workbench_feedback as feedback_module +import workbench_db as workbench from workbench_feedback import get_scan_feedback from workbench_schema import MIGRATIONS, apply_migrations @@ -33,7 +35,9 @@ connection.execute("PRAGMA foreign_keys = ON") identity_migration = next(item for item in MIGRATIONS if item[0] == 31) initial = ( (*tuple(item for item in MIGRATIONS if item[0] <= 28), (30, *identity_migration[1:])) - if scenario == "migration" else MIGRATIONS + if scenario == "migration" else + tuple(item for item in MIGRATIONS if item[0] <= 30) + if scenario == "null-history" else MIGRATIONS ) apply_migrations(connection, initial, lambda: timestamp, lambda database: None) paths = {} @@ -55,12 +59,19 @@ def add_target(name, stored, live=None, relative=".", birth=1_000_000_000): details[path] = state.GitRepositoryIdentity( live or stored or "repository-current", relative, str(root / "common"), 11, 22, birth ) - connection.execute( - "INSERT INTO security_targets " - "(id, current_path, display_name, created_at, updated_at, repository_identity) " - "VALUES (?, ?, ?, ?, ?, ?)", - (name, path, name, timestamp, timestamp, stored), - ) + if state.supports_repository_identity(connection): + connection.execute( + "INSERT INTO security_targets " + "(id, current_path, display_name, created_at, updated_at, repository_identity) " + "VALUES (?, ?, ?, ?, ?, ?)", + (name, path, name, timestamp, timestamp, stored), + ) + else: + connection.execute( + "INSERT INTO security_targets " + "(id, current_path, display_name, created_at, updated_at) VALUES (?, ?, ?, ?, ?)", + (name, path, name, timestamp, timestamp), + ) return details[path] @@ -210,6 +221,7 @@ with ExitStack() as stack: before = add_finding("persisted-scan", "before-review") after = add_finding("legacy-scan", "after-review", closed=True) add_finding("legacy-second", "legacy-open") + add_finding("unrelated-scan", "unrelated-finding", closed=True) connection.execute( "INSERT INTO scan_comparisons VALUES (?, ?, ?, ?, ?)", ("persisted-scan", "legacy-scan", "{}", timestamp, timestamp), @@ -233,9 +245,22 @@ with ExitStack() as stack: indexed = findings("requested") indexing_probes = dict(probes) probes.clear() - feedback = get_scan_feedback( - connection, connection.execute("SELECT * FROM scans WHERE id = 'requested-scan'").fetchone() - ) + feedback_scope = [] + feedback_queries = [] + def scoped_index(database, **kwargs): + feedback_scope.extend(sorted(kwargs["target_ids"])) + return indexes._indexed_findings(database, **kwargs) + connection.set_trace_callback(feedback_queries.append) + with patch.object(feedback_module, "_indexed_findings", side_effect=scoped_index): + feedback = get_scan_feedback( + connection, connection.execute("SELECT * FROM scans WHERE id = 'requested-scan'").fetchone() + ) + connection.set_trace_callback(None) + index_queries = [query for query in feedback_queries if any(marker in query for marker in ( + "SELECT before_scans.target_id AS before_target_id", + "SELECT scans.target_id, scans.id", + "occurrences.id AS occurrence_id", + ))] feedback_probes = dict(probes) print(json.dumps({ "scans": scans, @@ -244,6 +269,10 @@ with ExitStack() as stack: "aliases": sorted(indexes.repository_target_ids(connection, "requested")), "findings": indexed, "feedback": [row["findingId"] for row in feedback["falsePositives"]], + "feedbackScope": feedback_scope, + "feedbackIndexQueriesScoped": len(index_queries) == 3 and all( + "target_id IN (" in query for query in index_queries + ), "listingRequestedProbes": listing_probes.get(paths["requested"], 0), "matchingRequestedProbes": matching_probes.get(paths["requested"], 0), "matchingUnrelatedProbes": matching_probes.get(paths["unrelated"], 0), @@ -257,6 +286,74 @@ with ExitStack() as stack: "SELECT repository_identity FROM security_targets WHERE id = 'changed'" ).fetchone()[0], })) + elif scenario == "lineage": + for name, stored, live in [ + ("requested", "repository-current", None), + ("removed", "repository-current", None), + ("legacy", None, "repository-current"), + ("unverified", None, "repository-current"), + ("clone", "repository-other", None), + ("scope", "repository-current-scope", None), + ("changed", "repository-previous", "repository-current"), + ]: + add_target(name, stored, live, relative="service" if name == "scope" else ".") + add_scan(name + "-scan", name, "missing" if name == "unverified" else "current") + missing.add(paths["removed"]) + scan_dir = root / "scan-output" + original_iterdir = Path.iterdir + stack.enter_context(patch.object(Path, "iterdir", lambda path: iter(()) if path == scan_dir else original_iterdir(path))) + stack.enter_context(patch.object(workbench, "require_target", return_value=Path(paths["requested"]))) + stack.enter_context(patch.object(workbench, "require_scannable_target")) + stack.enter_context(patch.object(workbench, "require_canonical_scan_directory", return_value=scan_dir)) + stack.enter_context(patch.object(workbench, "directory_snapshot_regular_file_count", return_value=0)) + stack.enter_context(patch.object(workbench, "scan_target_identity", return_value={})) + stack.enter_context(patch.object(workbench, "require_uuid", side_effect=lambda value, label: value)) + stack.enter_context(patch.object(workbench, "archive_scan")) + class LineageAccepted(Exception): + pass + stack.enter_context(patch.object(workbench, "insert_running_scan", side_effect=LineageAccepted)) + connection.commit() + accepted = {} + for name in paths: + args = argparse.Namespace( + repository=paths["requested"], scan_dir=str(scan_dir), parent_scan_id=name + "-scan", + recipe_json=json.dumps({"repository": paths["requested"], "mode": "standard", "config": {}, "target": {"kind": "repository", "paths": []}}), + ) + try: + workbench.register_cli_scan(connection, args) + except LineageAccepted: + accepted[name] = True + except SystemExit: + accepted[name] = False + print(json.dumps({"accepted": accepted, "scanCount": connection.execute("SELECT COUNT(*) FROM scans").fetchone()[0]})) + elif scenario == "null-history": + newer_birth = state._timestamp_ns("2026-08-02T00:00:00Z") + for name, birth in [ + ("unchanged", 1_000_000_000), ("newer", newer_birth), + ("invalid-time", 1_000_000_000), ("no-history", newer_birth), + ]: + add_target(name, None, "current-" + name, birth=birth) + if name != "no-history": + add_scan(name + "-scan", name, started="invalid" if name == "invalid-time" else timestamp) + apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) + add_target("replacement-alias", "current-newer", birth=newer_birth) + add_scan("replacement-alias-scan", "replacement-alias", started="2026-08-03T00:00:00Z") + errors = {} + for name in ("unchanged", "newer", "invalid-time", "no-history"): + try: + state.ensure_security_target(connection, paths[name]) + except SystemExit as error: + errors[name] = str(error) + print(json.dumps({ + "stored": {row["id"]: row["repository_identity"] for row in connection.execute( + "SELECT id, repository_identity FROM security_targets" + )}, + "registrationErrors": sorted(errors), + "aliases": sorted(indexes.repository_target_ids(connection, "replacement-alias")), + "sameCheckoutMetadata": connection.execute( + "SELECT target_inode FROM scans WHERE id = 'newer-scan'" + ).fetchone()[0] == metadata[paths["newer"]].st_ino, + })) else: stack.enter_context(patch.object(os.path, "normcase", lambda value: os.fspath(value).lower())) originals = {} @@ -347,6 +444,8 @@ test("reuses verified legacy aliases and probes each saved target once per reque findings.find((finding) => finding["findingId"] === "legacy-open"), ).toBeDefined(); expect(result["feedback"]).toEqual(["after-review"]); + expect(result["feedbackScope"]).toEqual(result["aliases"]); + expect(result["feedbackIndexQueriesScoped"]).toBe(true); expect(result["listingRequestedProbes"]).toBe(1); expect(result["matchingRequestedProbes"]).toBe(1); expect(result["matchingUnrelatedProbes"]).toBe(1); @@ -369,3 +468,33 @@ test("upgrades only independently verified pre-release repository hashes", () => Array.from({ length: 31 }, (_, index) => index + 1), ); }); + +test("does not assign legacy history to a newer or indeterminate Git generation", () => { + const result = run("null-history"); + + expect(result["sameCheckoutMetadata"]).toBe(true); + expect(result["stored"]).toEqual({ + unchanged: "current-unchanged", + newer: null, + "invalid-time": null, + "no-history": "current-no-history", + "replacement-alias": "current-newer", + }); + expect(result["registrationErrors"]).toEqual(["invalid-time", "newer"]); + expect(result["aliases"]).toEqual(["replacement-alias"]); +}); + +test("admits rerun lineage only through the verified repository and exact scope", () => { + const result = run("lineage"); + + expect(result["accepted"]).toEqual({ + requested: true, + removed: true, + legacy: true, + unverified: false, + clone: false, + scope: false, + changed: false, + }); + expect(result["scanCount"]).toBe(7); +}); diff --git a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts index d6e15ab4c..bb2b5e269 100644 --- a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts +++ b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts @@ -74,6 +74,7 @@ import sqlite3 import subprocess import sys from dataclasses import replace +from datetime import datetime, timezone from pathlib import Path from types import SimpleNamespace from unittest.mock import patch @@ -97,7 +98,7 @@ from workbench_target_state import ( scenario = sys.argv[2] root, repository, worktree, clone = map(Path, sys.argv[3:7]) -timestamp = "2026-08-01T00:00:00Z" +timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") connection = sqlite3.connect(":memory:") connection.row_factory = sqlite3.Row connection.execute("PRAGMA foreign_keys = ON") diff --git a/sdk/typescript/tests-ts/workbench-scan-history.test.ts b/sdk/typescript/tests-ts/workbench-scan-history.test.ts index 65529e871..1bdbd1d28 100644 --- a/sdk/typescript/tests-ts/workbench-scan-history.test.ts +++ b/sdk/typescript/tests-ts/workbench-scan-history.test.ts @@ -31,7 +31,16 @@ test("loads each scan's matching findings once across historical batches", () => "connection.set_trace_callback(queries.append)", "backfilled = []", "result = history.list_unmatched_scan_pairs(connection, argparse.Namespace(repository=sys.argv[2], force=False), backfill_finding_details=lambda _connection, scan: backfilled.append(scan['id']), read_coverage=lambda _scan: {})", - "print(json.dumps({'result': result, 'backfilled': backfilled, 'findingQueries': sum('FROM finding_occurrences AS occurrences' in query for query in queries)}))", + "finding_queries = sum('FROM finding_occurrences AS occurrences' in query for query in queries)", + "def planned(focus=None):", + " value = history.list_unmatched_scan_pairs(connection, argparse.Namespace(repository=sys.argv[2], force=False, after_scan_id=focus), backfill_finding_details=lambda *_: None, read_coverage=lambda _: {})", + " return {'batches': [{'after': batch['afterScanId'], 'before': [scan['scanId'] for scan in batch['beforeScans']]} for batch in value['batches']], 'skipped': value['skippedPairs']}", + "connection.execute(\"UPDATE scans SET status = 'running' WHERE id = 'scan-0'\")", + "later_completed_first = planned('scan-2')", + "connection.execute(\"UPDATE scans SET status = 'complete' WHERE id = 'scan-0'\")", + "earlier_completed_last = planned('scan-0')", + "connection.execute(\"INSERT INTO scan_comparisons VALUES ('scan-2', 'scan-0')\")", + "print(json.dumps({'result': result, 'backfilled': backfilled, 'findingQueries': finding_queries, 'laterCompletedFirst': later_completed_first, 'earlierCompletedLast': earlier_completed_last, 'reverseCached': planned('scan-0'), 'chronological': planned()}))", ].join("\n"); const result = spawnSync( @@ -52,6 +61,25 @@ test("loads each scan's matching findings once across historical batches", () => expect(JSON.parse(result.stdout)).toMatchObject({ backfilled: ["scan-0", "scan-1", "scan-2"], findingQueries: 3, + laterCompletedFirst: { + batches: [{ after: "scan-2", before: ["scan-1"] }], + skipped: 0, + }, + earlierCompletedLast: { + batches: [{ after: "scan-0", before: ["scan-1", "scan-2"] }], + skipped: 0, + }, + reverseCached: { + batches: [{ after: "scan-0", before: ["scan-1"] }], + skipped: 1, + }, + chronological: { + batches: [ + { after: "scan-1", before: ["scan-0"] }, + { after: "scan-2", before: ["scan-1"] }, + ], + skipped: 1, + }, result: { scanCount: 3, batches: [ From d7b4b648a989174636f91dd9d24468470bf9f280 Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sun, 16 Aug 2026 01:52:09 -0700 Subject: [PATCH 09/21] fix(workbench): quarantine unproved prerelease identities --- .../scripts/workbench_target_state.py | 63 +++++++++------- .../tests-ts/workbench-identity-cache.test.ts | 74 ++++++++++++++++++- .../workbench-target-migration.test.ts | 7 +- 3 files changed, 110 insertions(+), 34 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index 9540e1920..c97ab65b0 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -454,38 +454,51 @@ def _repository_predates_history( def normalize_pre_release_repository_identities(connection: sqlite3.Connection) -> None: - """Upgrade only known old hashes whose recorded generation is still present.""" + """Retain only individually established pre-release identity bindings.""" if not supports_repository_identity(connection): return - for target in connection.execute( - "SELECT id, current_path, created_at, repository_identity FROM security_targets " - "WHERE repository_identity IS NOT NULL" - ).fetchall(): + identities = RepositoryIdentityCache(connection) + targets = connection.execute( + "SELECT id AS target_id, current_path AS target_path, created_at, repository_identity " + "FROM security_targets WHERE repository_identity IS NOT NULL" + ).fetchall() + states = {target["target_id"]: identities.for_row(target) for target in targets} + anchors = { + state.stored_identity: state.repository + for state in states.values() + if state.repository is not None + and state.verified_identity == state.stored_identity + } + for target in targets: stored = target["repository_identity"] - state = _inspect_repository_target( - connection, target["id"], target["current_path"], stored - ) - identity = state.repository - if ( - identity is None - or identity.value == stored - or not state.strict_owner_matches - or stored not in _pre_release_repository_identities(identity) - ): - continue + state = states[target["target_id"]] scans = connection.execute( "SELECT started_at, created_at FROM scans WHERE target_id = ? OR target_path = ?", - (target["id"], target["current_path"]), + (target["target_id"], target["target_path"]), ).fetchall() - if not _repository_predates_history( - identity, scans, empty_timestamp=target["created_at"] + anchor = anchors.get(stored) + identity = state.repository + replacement = None + if anchor is not None: + if _repository_predates_history( + anchor, scans, empty_timestamp=target["created_at"] + ): + replacement = stored + elif ( + identity is not None + and state.strict_owner_matches + and stored in _pre_release_repository_identities(identity) + and _repository_predates_history( + identity, scans, empty_timestamp=target["created_at"] + ) ): - continue - connection.execute( - "UPDATE security_targets SET repository_identity = ? " - "WHERE id = ? AND repository_identity = ?", - (identity.value, target["id"], stored), - ) + replacement = identity.value + if replacement != stored: + connection.execute( + "UPDATE security_targets SET repository_identity = ? " + "WHERE id = ? AND repository_identity = ?", + (replacement, target["target_id"], stored), + ) def backfill_repository_identities(connection: sqlite3.Connection) -> None: diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts index 859ddaa93..970f0635f 100644 --- a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -35,7 +35,7 @@ connection.execute("PRAGMA foreign_keys = ON") identity_migration = next(item for item in MIGRATIONS if item[0] == 31) initial = ( (*tuple(item for item in MIGRATIONS if item[0] <= 28), (30, *identity_migration[1:])) - if scenario == "migration" else + if scenario in ("migration", "v30-current") else tuple(item for item in MIGRATIONS if item[0] <= 30) if scenario == "null-history" else MIGRATIONS ) @@ -75,7 +75,7 @@ def add_target(name, stored, live=None, relative=".", birth=1_000_000_000): return details[path] -def add_scan(scan_id, target, owner="current", started=timestamp): +def add_scan(scan_id, target, owner="current", started=timestamp, created=timestamp): path = paths[target] device, inode = metadata[path].st_dev, metadata[path].st_ino if owner == "missing": @@ -94,7 +94,7 @@ def add_scan(scan_id, target, owner="current", started=timestamp): "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (scan_id, "workspace-" + scan_id, path, target, device, inode, "synthetic", ".", "standard", str(root / "scans" / scan_id), "complete", "reporting", - started, timestamp, timestamp, timestamp), + started, created, created, created), ) connection.execute( "INSERT INTO scan_progress (scan_id, updated_at) VALUES (?, ?)", @@ -286,6 +286,48 @@ with ExitStack() as stack: "SELECT repository_identity FROM security_targets WHERE id = 'changed'" ).fetchone()[0], })) + elif scenario == "v30-current": + generation_birth = state._timestamp_ns("2026-08-02T00:00:00Z") + later = "2026-08-03T00:00:00Z" + for name in ("current-owner", "old-owner", "removed-old", "removed-valid", "invalid-time"): + add_target(name, "current-generation", birth=generation_birth) + recorded = later if name in ("current-owner", "removed-valid") else timestamp + add_scan(name + "-scan", name, started="invalid" if name == "invalid-time" else recorded, created=recorded) + add_target("unverified-anchor", "unverified-current") + add_scan("unverified-anchor-scan", "unverified-anchor", owner="missing") + add_target("opaque-missing", "opaque-current") + add_scan("opaque-missing-scan", "opaque-missing") + add_target("valid-scope", "scope-current", relative="service") + add_scan("valid-scope-scan", "valid-scope") + missing.update(paths[name] for name in ("removed-old", "removed-valid", "opaque-missing")) + for name in ("current-owner", "old-owner", "removed-old", "removed-valid"): + add_finding(name + "-scan", name + "-finding", closed=name != "current-owner") + connection.execute( + "INSERT INTO scan_artifacts VALUES (?, ?, ?, ?)", + (name + "-scan", "findings", str(root / name / "findings.json"), timestamp), + ) + tables = ("scans", "findings", "finding_occurrences", "finding_triage", "scan_artifacts") + retained = {table: [tuple(row) for row in connection.execute("SELECT * FROM " + table)] for table in tables} + target_ids = sorted(paths) + apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) + state.backfill_repository_identities(connection) + try: + state.ensure_security_target(connection, paths["old-owner"]) + except SystemExit: + old_registration_rejected = True + else: + old_registration_rejected = False + print(json.dumps({ + "stored": {row["id"]: row["repository_identity"] for row in connection.execute( + "SELECT id, repository_identity FROM security_targets" + )}, + "recordsPreserved": retained == {table: [tuple(row) for row in connection.execute("SELECT * FROM " + table)] for table in tables}, + "targetIdsPreserved": target_ids == sorted(row["id"] for row in connection.execute("SELECT id FROM security_targets")), + "aliases": sorted(indexes.repository_target_ids(connection, "current-owner")), + "removedExact": listed("removed-old"), + "visibleFindings": sorted(row["findingId"] for row in findings("current-owner")), + "oldRegistrationRejected": old_registration_rejected, + })) elif scenario == "lineage": for name, stored, live in [ ("requested", "repository-current", None), @@ -382,7 +424,7 @@ with ExitStack() as stack: missing.add(paths[name]) expected[name] = identity.value if name in { "old-basic", "old-description", "current", "no-scans", "scope-upper", "scope-lower" - } else stored + } else None apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) first = { row["id"]: row["repository_identity"] @@ -498,3 +540,27 @@ test("admits rerun lineage only through the verified repository and exact scope" }); expect(result["scanCount"]).toBe(7); }); + +test("quarantines unproved public-v30 bindings without discarding historical records", () => { + const result = run("v30-current"); + + expect(result["stored"]).toEqual({ + "current-owner": "current-generation", + "old-owner": null, + "removed-old": null, + "removed-valid": "current-generation", + "invalid-time": null, + "unverified-anchor": null, + "opaque-missing": null, + "valid-scope": "scope-current", + }); + expect(result["recordsPreserved"]).toBe(true); + expect(result["targetIdsPreserved"]).toBe(true); + expect(result["aliases"]).toEqual(["current-owner", "removed-valid"]); + expect(result["removedExact"]).toEqual(["removed-old-scan"]); + expect(result["visibleFindings"]).toEqual([ + "current-owner-finding", + "removed-valid-finding", + ]); + expect(result["oldRegistrationRejected"]).toBe(true); +}); diff --git a/sdk/typescript/tests-ts/workbench-target-migration.test.ts b/sdk/typescript/tests-ts/workbench-target-migration.test.ts index 9c793ee5e..03a26c1c6 100644 --- a/sdk/typescript/tests-ts/workbench-target-migration.test.ts +++ b/sdk/typescript/tests-ts/workbench-target-migration.test.ts @@ -349,7 +349,7 @@ describe("stable workbench target migration", () => { ], [ "pre-release-identity-version", - "preserves an identity recorded under the pre-release migration number", + "quarantines an unverifiable pre-release identity without changing its target", ], ] as const)("%s: %s", (scenario) => { const python = @@ -397,10 +397,7 @@ describe("stable workbench target migration", () => { }, repositoryIdentityColumnIsNullable: true, repositoryIdentityIndexIsUnique: false, - targetIdentity: - scenario === "pre-release-identity-version" - ? "synthetic-identity" - : null, + targetIdentity: null, targetId: "target-existing", teamOnlyPublicationIndexes: [ "finding_publications_team_only_external_issue", From c93dd737346eb9836eb33c4f575e39867337a31b Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sun, 16 Aug 2026 03:25:03 -0700 Subject: [PATCH 10/21] fix(workbench): order automatic history comparisons --- .../scripts/workbench_scan_history.py | 39 ++++++-------- .../tests-ts/workbench-identity-cache.test.ts | 51 +++++++++++++++++++ .../tests-ts/workbench-scan-history.test.ts | 34 +++++++++++-- 3 files changed, 95 insertions(+), 29 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py index 83310db52..46a333945 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py @@ -17,6 +17,7 @@ from workbench_scan_usage import stored_scan_cost_fields from workbench_target_state import ( RepositoryIdentityCache, + _timestamp_ns, supports_repository_identity, ) @@ -89,22 +90,13 @@ def list_scans( ] if identity_matches else ["0"] requested_identity = requested.verified_identity if requested_identity is not None: - for target_id, target in identities.targets.items(): + for target_id in identities.targets: if target_id == requested.target_id or identities.group(target_id) != ( "repository", requested_identity ): continue - candidate = identities.for_row(target) - if candidate.resolved_path is None: - continue - repository_clauses.append( - _owned_scan_clause( - Path(candidate.target_path), - check_ownership and not candidate.relaxed_directory_ownership, - values, - target_id=target_id, - ) - ) + repository_clauses.append("scans.target_id = ?") + values.append(target_id) clauses.append(f"({' OR '.join(repository_clauses)})") if args is not None and args.scan_root: scan_root = str(Path(args.scan_root).expanduser().resolve()) @@ -222,15 +214,9 @@ def _owned_scan_clause( target: Path, check_ownership: bool, values: list[Any], - *, - target_id: str | None = None, ) -> str: - if target_id is None: - values.append(str(target)) - target_clause = "scans.target_path = ?" - else: - values.append(target_id) - target_clause = "scans.target_id = ?" + values.append(str(target)) + target_clause = "scans.target_path = ?" if not check_ownership: return target_clause try: @@ -251,6 +237,12 @@ def _owned_scan_clause( ) +def _scan_completion_order(scan: sqlite3.Row) -> tuple[int, str]: + completed_at = scan["completed_at"] if "completed_at" in scan.keys() else None + timestamp = _timestamp_ns(completed_at) + if timestamp is None: + timestamp = _timestamp_ns(scan["started_at"]) + return (timestamp if timestamp is not None else 0, scan["id"]) def list_unmatched_scan_pairs( @@ -298,16 +290,15 @@ def list_unmatched_scan_pairs( focus_scan_id = getattr(args, "after_scan_id", None) if focus_scan_id is not None and not any(scan["id"] == focus_scan_id for scan in selected): raise SystemExit("The scan to match is not a completed scan in this repository.") + if focus_scan_id is not None: + available.sort(key=_scan_completion_order) batches = [] skipped = 0 matching_findings: dict[str, list[dict[str, Any]]] = {} for index, after in enumerate(available): if focus_scan_id is not None and after["id"] != focus_scan_id: continue - candidates = ( - available[:index] if focus_scan_id is None - else [before for before in available if before["id"] != focus_scan_id] - ) + candidates = available[:index] previous = [ before for before in candidates diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts index 970f0635f..09109dbeb 100644 --- a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -286,6 +286,40 @@ with ExitStack() as stack: "SELECT repository_identity FROM security_targets WHERE id = 'changed'" ).fetchone()[0], })) + elif scenario == "persisted-alias": + for name, stored, live in [ + ("requested", "repository-current", None), + ("reused", "repository-current", "repository-other"), + ("legacy", None, "repository-current"), + ("unverified", None, "repository-current"), + ("unrelated", "repository-other", None), + ]: + add_target(name, stored, live) + add_scan(name + "-scan", name, "missing" if name == "unverified" else "current") + metadata[paths["reused"]].st_ino += 1000 + add_finding("requested-scan", "current-finding") + add_finding("reused-scan", "historical-finding", closed=True) + scans = listed("requested") + reused_listing_probes = probes[paths["reused"]] + matching = history.list_unmatched_scan_pairs( + connection, argparse.Namespace(repository=paths["requested"], force=False), + backfill_finding_details=lambda *_: None, read_coverage=lambda _: {}, + ) + feedback = get_scan_feedback( + connection, connection.execute("SELECT * FROM scans WHERE id = 'requested-scan'").fetchone() + ) + print(json.dumps({ + "scans": scans, + "replacementRequest": listed("reused"), + "aliases": sorted(indexes.repository_target_ids(connection, "requested")), + "findings": sorted(row["findingId"] for row in findings("requested")), + "feedback": [row["findingId"] for row in feedback["falsePositives"]], + "matchingCount": matching["scanCount"], + "reusedListingProbes": reused_listing_probes, + "stored": connection.execute( + "SELECT repository_identity FROM security_targets WHERE id = 'reused'" + ).fetchone()[0], + })) elif scenario == "v30-current": generation_birth = state._timestamp_ns("2026-08-02T00:00:00Z") later = "2026-08-03T00:00:00Z" @@ -498,6 +532,23 @@ test("reuses verified legacy aliases and probes each saved target once per reque expect(result["changedStored"]).toBe("repository-previous"); }); +test("keeps authenticated historical aliases visible without trusting a replacement checkout", () => { + const result = run("persisted-alias"); + + expect(result["scans"]).toEqual([ + "legacy-scan", + "requested-scan", + "reused-scan", + ]); + expect(result["replacementRequest"]).toEqual([]); + expect(result["aliases"]).toEqual(["legacy", "requested", "reused"]); + expect(result["findings"]).toEqual(["current-finding", "historical-finding"]); + expect(result["feedback"]).toEqual(["historical-finding"]); + expect(result["matchingCount"]).toBe(3); + expect(result["reusedListingProbes"]).toBe(0); + expect(result["stored"]).toBe("repository-current"); +}); + test("upgrades only independently verified pre-release repository hashes", () => { const result = run("migration"); diff --git a/sdk/typescript/tests-ts/workbench-scan-history.test.ts b/sdk/typescript/tests-ts/workbench-scan-history.test.ts index 1bdbd1d28..46cd19880 100644 --- a/sdk/typescript/tests-ts/workbench-scan-history.test.ts +++ b/sdk/typescript/tests-ts/workbench-scan-history.test.ts @@ -17,7 +17,7 @@ test("loads each scan's matching findings once across historical batches", () => "connection.row_factory = sqlite3.Row", "connection.executescript('''", "CREATE TABLE security_targets (id TEXT, current_path TEXT);", - "CREATE TABLE scans (id TEXT, target_path TEXT, target_id TEXT, status TEXT, started_at TEXT);", + "CREATE TABLE scans (id TEXT, target_path TEXT, target_id TEXT, status TEXT, started_at TEXT, completed_at TEXT);", "CREATE TABLE scan_comparisons (before_scan_id TEXT, after_scan_id TEXT);", "CREATE TABLE finding_occurrences (id TEXT, finding_id TEXT, scan_id TEXT, details_json TEXT, remediation TEXT, severity TEXT, summary TEXT, title TEXT);", "CREATE TABLE finding_triage (occurrence_id TEXT, status TEXT, close_reason TEXT);", @@ -25,7 +25,7 @@ test("loads each scan's matching findings once across historical batches", () => "''')", "for index in range(3):", " scan = f'scan-{index}'", - " connection.execute('INSERT INTO scans VALUES (?, ?, NULL, ?, ?)', (scan, sys.argv[2], 'complete', str(index)))", + " connection.execute('INSERT INTO scans VALUES (?, ?, NULL, ?, ?, ?)', (scan, sys.argv[2], 'complete', f'2026-08-01T0{index}:00:00Z', f'2026-08-01T0{index + 3}:00:00Z'))", " connection.execute('INSERT INTO finding_occurrences VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (scan, scan, scan, '{}', 'fix', 'high', 'summary', 'title'))", "queries = []", "connection.set_trace_callback(queries.append)", @@ -35,12 +35,21 @@ test("loads each scan's matching findings once across historical batches", () => "def planned(focus=None):", " value = history.list_unmatched_scan_pairs(connection, argparse.Namespace(repository=sys.argv[2], force=False, after_scan_id=focus), backfill_finding_details=lambda *_: None, read_coverage=lambda _: {})", " return {'batches': [{'after': batch['afterScanId'], 'before': [scan['scanId'] for scan in batch['beforeScans']]} for batch in value['batches']], 'skipped': value['skippedPairs']}", - "connection.execute(\"UPDATE scans SET status = 'running' WHERE id = 'scan-0'\")", + "connection.execute(\"UPDATE scans SET status = 'running', completed_at = NULL WHERE id = 'scan-0'\")", "later_completed_first = planned('scan-2')", - "connection.execute(\"UPDATE scans SET status = 'complete' WHERE id = 'scan-0'\")", + "connection.execute(\"UPDATE scans SET status = 'complete', completed_at = '2026-08-01T06:00:00Z' WHERE id = 'scan-0'\")", "earlier_completed_last = planned('scan-0')", + "excludes_later_completion = planned('scan-2')", + "def automatic_pairs():", + " return [[before, batch['after']] for index in range(3) for batch in planned(f'scan-{index}')['batches'] for before in batch['before']]", + "completion_pairs = automatic_pairs()", "connection.execute(\"INSERT INTO scan_comparisons VALUES ('scan-2', 'scan-0')\")", - "print(json.dumps({'result': result, 'backfilled': backfilled, 'findingQueries': finding_queries, 'laterCompletedFirst': later_completed_first, 'earlierCompletedLast': earlier_completed_last, 'reverseCached': planned('scan-0'), 'chronological': planned()}))", + "reverse_cached, chronological = planned('scan-0'), planned()", + "connection.execute('DELETE FROM scan_comparisons')", + "connection.execute(\"UPDATE scans SET completed_at = CASE id WHEN 'scan-0' THEN NULL WHEN 'scan-1' THEN '2026-08-01T05:00:00+01:00' ELSE '2026-08-01T04:00:00Z' END\")", + "legacy = dict(connection.execute(\"SELECT * FROM scans WHERE id = 'scan-0'\").fetchone())", + "legacy_without_completed = {key: value for key, value in legacy.items() if key != 'completed_at'}", + "print(json.dumps({'result': result, 'backfilled': backfilled, 'findingQueries': finding_queries, 'laterCompletedFirst': later_completed_first, 'earlierCompletedLast': earlier_completed_last, 'excludesLaterCompletion': excludes_later_completion, 'completionPairs': completion_pairs, 'reverseCached': reverse_cached, 'chronological': chronological, 'tiedPairs': automatic_pairs(), 'legacyFallback': history._scan_completion_order(legacy) == history._scan_completion_order(legacy_without_completed)}))", ].join("\n"); const result = spawnSync( @@ -69,6 +78,21 @@ test("loads each scan's matching findings once across historical batches", () => batches: [{ after: "scan-0", before: ["scan-1", "scan-2"] }], skipped: 0, }, + excludesLaterCompletion: { + batches: [{ after: "scan-2", before: ["scan-1"] }], + skipped: 0, + }, + completionPairs: [ + ["scan-1", "scan-0"], + ["scan-2", "scan-0"], + ["scan-1", "scan-2"], + ], + tiedPairs: [ + ["scan-0", "scan-1"], + ["scan-0", "scan-2"], + ["scan-1", "scan-2"], + ], + legacyFallback: true, reverseCached: { batches: [{ after: "scan-0", before: ["scan-1"] }], skipped: 1, From 042fb828ff0b8e28f6265354f89d271c83dc2148 Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sun, 16 Aug 2026 04:21:18 -0700 Subject: [PATCH 11/21] fix(workbench): persist completion visibility order --- sdk/typescript/README.md | 4 +- .../_bundled_plugin/scripts/workbench_cli.py | 4 +- .../_bundled_plugin/scripts/workbench_db.py | 4 + .../scripts/workbench_native_indexes.py | 41 +++- .../scripts/workbench_scan_history.py | 7 +- .../scripts/workbench_schema.py | 70 +++++- .../scripts/workbench_target_state.py | 17 +- sdk/typescript/src/api.ts | 9 +- sdk/typescript/src/cli.ts | 32 ++- sdk/typescript/tests-ts/cli-workbench.test.ts | 93 +++----- .../tests-ts/workbench-identity-cache.test.ts | 221 +++++++++++++++++- .../workbench-target-migration.test.ts | 32 ++- 12 files changed, 419 insertions(+), 115 deletions(-) diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 8144731ee..8c1861ef0 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -668,7 +668,9 @@ const directPublication = await publishScan("/path/to/completed-scan", { `scans` or `scans list` lists scans for the current repository. Linked Git worktrees are discovered automatically and grouped when they share the selected state database. Findings indicate whether they were confirmed in the repository's -latest completed scan across those linked worktrees. Pass a repository path to +latest completed scan across those linked worktrees. Finding confirmation and +automatic matching use the order in which completed scans become visible in the +workbench; sealed report timestamps are unchanged. Pass a repository path to inspect another checkout. `--scan-root DIR` only filters scans already recorded in that database by their artifact directory; it never imports scan results from another state directory. `scans show` opens the latest completed scan for the diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py b/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py index 13276e7e5..fa508f2a8 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py @@ -170,7 +170,9 @@ def parse_args(description: str) -> argparse.Namespace: list_global_findings.add_argument("--query") list_global_findings.add_argument("--severity", choices=FINDING_SEVERITIES) list_global_findings.add_argument("--status", choices=FINDING_STATUSES) - list_global_findings.add_argument("--target-id") + finding_repository = list_global_findings.add_mutually_exclusive_group() + finding_repository.add_argument("--target-id") + finding_repository.add_argument("--repository") list_global_findings.add_argument("--offset", type=non_negative_int, default=0) list_global_findings.add_argument("--limit", type=positive_int, default=FINDINGS_PAGE_MAX) list_repositories = subparsers.add_parser("list-repositories") diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index ab540df25..e094baa80 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -885,6 +885,10 @@ def start_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict "This Codex thread already has an active Deep Scan for the selected " "target and scope. Rejoin that scan instead of starting another one." ) + if ensure_security_target(connection, str(current_target)) != workspace["target_id"]: + raise SystemExit( + "The saved workspace no longer matches the selected repository target." + ) insert_running_scan( connection, scan_id=scan_id, diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py index 203b17a21..e71b9fc5c 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py @@ -33,9 +33,15 @@ def list_global_findings( limit = min(args.limit, FINDINGS_PAGE_MAX) query = args.query.strip().casefold() if args.query else "" identities = RepositoryIdentityCache(connection) - target_ids = ( - None if args.target_id is None else identities.target_ids(args.target_id) - ) + repository = getattr(args, "repository", None) + if repository is not None: + target_ids = identities.target_ids_for_path( + str(Path(repository).expanduser().resolve()) + ) + else: + target_ids = ( + None if args.target_id is None else identities.target_ids(args.target_id) + ) findings = ( row for row in _indexed_findings(connection, identities=identities, target_ids=target_ids) @@ -149,17 +155,28 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: if before != after: parents[after] = before + completion_column = ( + "scans.completion_sequence" + if "completion_sequence" in identities.scan_columns else "NULL" + ) + completed_at_column = ( + "scans.completed_at" if "completed_at" in identities.scan_columns else "NULL" + ) latest_scan_by_repository = { identities.group(row["target_id"]): row["id"] - for row in connection.execute( - f""" - SELECT scans.target_id, scans.id - FROM scans - JOIN security_targets AS targets ON targets.id = scans.target_id - WHERE scans.status = 'complete' AND {target_filter} - ORDER BY scans.started_at, scans.id - """, - target_values, + for row in sorted( + connection.execute( + f""" + SELECT scans.target_id, scans.id, scans.started_at, + {completed_at_column} AS completed_at, + {completion_column} AS completion_sequence + FROM scans + JOIN security_targets AS targets ON targets.id = scans.target_id + WHERE scans.status = 'complete' AND {target_filter} + """, + target_values, + ), + key=scan_history._scan_completion_order, ) } diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py index 46a333945..0460c4823 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py @@ -237,12 +237,15 @@ def _owned_scan_clause( ) -def _scan_completion_order(scan: sqlite3.Row) -> tuple[int, str]: +def _scan_completion_order(scan: sqlite3.Row) -> tuple[int, int, str]: + sequence = scan["completion_sequence"] if "completion_sequence" in scan.keys() else None + if sequence is not None: + return (1, sequence, scan["id"]) completed_at = scan["completed_at"] if "completed_at" in scan.keys() else None timestamp = _timestamp_ns(completed_at) if timestamp is None: timestamp = _timestamp_ns(scan["started_at"]) - return (timestamp if timestamp is not None else 0, scan["id"]) + return (0, timestamp if timestamp is not None else 0, scan["id"]) def list_unmatched_scan_pairs( diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py index 345f91595..4365feafe 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py @@ -692,6 +692,34 @@ CREATE INDEX security_targets_by_repository_identity ON security_targets(repository_identity); + + ALTER TABLE scans + ADD COLUMN completion_sequence INTEGER CHECK (completion_sequence >= 1); + + CREATE UNIQUE INDEX scans_completion_sequence + ON scans(completion_sequence); + + CREATE TRIGGER scans_assign_inserted_completion_sequence + AFTER INSERT ON scans + WHEN NEW.status = 'complete' AND NEW.completion_sequence IS NULL + BEGIN + UPDATE scans + SET completion_sequence = ( + SELECT COALESCE(MAX(completion_sequence), 0) + 1 FROM scans + ) + WHERE id = NEW.id AND completion_sequence IS NULL; + END; + + CREATE TRIGGER scans_assign_updated_completion_sequence + AFTER UPDATE OF status ON scans + WHEN NEW.status = 'complete' AND NEW.completion_sequence IS NULL + BEGIN + UPDATE scans + SET completion_sequence = ( + SELECT COALESCE(MAX(completion_sequence), 0) + 1 FROM scans + ) + WHERE id = NEW.id AND completion_sequence IS NULL; + END; """, ), ) @@ -1300,15 +1328,41 @@ def repair_repository_identity_migration(connection: sqlite3.Connection) -> bool "SELECT 1 FROM sqlite_master WHERE type = 'index' " "AND name = 'security_targets_by_repository_identity'" ).fetchone() - if "repository_identity" in columns and index_exists is not None: - return False + identity_changed = "repository_identity" not in columns or index_exists is None + migration_sql = next(sql for version, _, sql in MIGRATIONS if version == 31) + for statement in sql_statements(migration_sql): + if statement.startswith("ALTER TABLE security_targets"): + add_column_if_missing(connection, "security_targets", "repository_identity", "TEXT") + continue + if statement.startswith("ALTER TABLE scans"): + add_column_if_missing( + connection, "scans", "completion_sequence", + "INTEGER CHECK (completion_sequence >= 1)", + ) + continue + for prefix in ("CREATE UNIQUE INDEX ", "CREATE INDEX ", "CREATE TRIGGER "): + if statement.startswith(prefix): + statement = statement.replace(prefix, f"{prefix}IF NOT EXISTS ", 1) + break + connection.execute(statement) - add_column_if_missing(connection, "security_targets", "repository_identity", "TEXT") - connection.execute( - "CREATE INDEX IF NOT EXISTS security_targets_by_repository_identity " - "ON security_targets(repository_identity)" - ) - return True + from workbench_scan_history import _scan_completion_order + + sequence = connection.execute( + "SELECT COALESCE(MAX(completion_sequence), 0) FROM scans" + ).fetchone()[0] + unsequenced = connection.execute( + "SELECT id, started_at, completed_at FROM scans " + "WHERE status = 'complete' AND completion_sequence IS NULL" + ).fetchall() + for scan in sorted(unsequenced, key=_scan_completion_order): + sequence += 1 + connection.execute( + "UPDATE scans SET completion_sequence = ? " + "WHERE id = ? AND completion_sequence IS NULL", + (sequence, scan["id"]), + ) + return identity_changed def add_column_if_missing( diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index c97ab65b0..6bd632d9e 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -389,12 +389,21 @@ def target_ids(self, target_id: str) -> set[str]: target = self.targets.get(target_id) if not self.supports_identity or target is None: return {target_id} - requested = self.for_row(target) + return self._target_ids_for_state(self.for_row(target)) + + def target_ids_for_path(self, target_path: str) -> set[str]: + return self._target_ids_for_state(self.for_path(target_path)) + + def _target_ids_for_state(self, requested: RepositoryTargetState) -> set[str]: if not requested.ownership_matches: return set() - group = self.group(target_id) - if group[0] == "target": - return {target_id} + identity = ( + requested.stored_identity or requested.verified_identity + if self.supports_identity else None + ) + if identity is None: + return {requested.target_id} if requested.target_id else set() + group = ("repository", identity) return {candidate for candidate in self.targets if self.group(candidate) == group} def origin(self, state: RepositoryTargetState) -> tuple[str, str] | None: diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts index 378cc07a3..d29f3845c 100644 --- a/sdk/typescript/src/api.ts +++ b/sdk/typescript/src/api.ts @@ -1923,16 +1923,19 @@ export class CodexSecurity { export async function listRepositoryFindings( workbench: (args: readonly string[]) => Promise, - targetId: string, + target: string | { repository: string }, status: "open" | "all" = "open", ): Promise { const findings: JsonObject[] = []; + const selector = + typeof target === "string" + ? ["--target-id", target] + : ["--repository", target.repository]; let offset: number | undefined; do { const page = await workbench([ "list-global-findings", - "--target-id", - targetId, + ...selector, ...(status === "open" ? ["--status", "open"] : []), ...(offset === undefined ? [] : ["--offset", String(offset)]), ]); diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index b6b1437ec..5a63ffdcd 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -1083,12 +1083,16 @@ export async function main( let renderedHistory: string | undefined; let renderedPublication: string | undefined; const history = async ( - args: readonly string[], + args: readonly string[] | (() => Promise), select: (value: JsonObject) => JsonObject | Promise = (value) => value, ): Promise => { try { - return await select(await dependencies.runWorkbench(args)); + return await select( + await (typeof args === "function" + ? args() + : dependencies.runWorkbench(args)), + ); } catch (error) { errorOutput.write(`codex-security: ${errorMessage(error)}\n`); exitCode = 2; @@ -1236,23 +1240,13 @@ export async function main( args.repository ?? ".", ); return presentHistory( - await history( - ["list-scans", "--repository", repository], - async (value): Promise => { - const scans = value["scans"] as JsonObject[]; - const target = - scans.find((scan) => scan["targetPath"] === repository) ?? - scans[0]; - const findings = - target === undefined - ? [] - : await listRepositoryFindings( - dependencies.runWorkbench, - target["targetId"] as string, - ); - return { repository, findings: findings ?? [] }; - }, - ), + await history(async () => ({ + repository, + findings: + (await listRepositoryFindings(dependencies.runWorkbench, { + repository, + })) ?? [], + })), "findings", format, { repository }, diff --git a/sdk/typescript/tests-ts/cli-workbench.test.ts b/sdk/typescript/tests-ts/cli-workbench.test.ts index d56c34437..1fa7c7e3f 100644 --- a/sdk/typescript/tests-ts/cli-workbench.test.ts +++ b/sdk/typescript/tests-ts/cli-workbench.test.ts @@ -18,7 +18,6 @@ describe("CLI workbench", () => { const stdout = capture(); const calls: Array = []; const responses: JsonObject[] = [ - { scans: [{ targetId: "selected", targetPath: repository }] }, { findings: [{ title: "Finding 1" }], nextOffset: 1 }, { findings: [{ title: "Finding 2" }], nextOffset: null }, ]; @@ -32,15 +31,14 @@ describe("CLI workbench", () => { }), ), ).toBe(0); - expect(calls[0]).toEqual(["list-scans", "--repository", repository]); - expect(calls[1]).toEqual([ + expect(calls[0]).toEqual([ "list-global-findings", - "--target-id", - "selected", + "--repository", + repository, "--status", "open", ]); - expect(calls[2]).toEqual([...calls[1]!, "--offset", "1"]); + expect(calls[1]).toEqual([...calls[0]!, "--offset", "1"]); expect(JSON.parse(stdout.text())).toEqual({ repository, findings: [{ title: "Finding 1" }, { title: "Finding 2" }], @@ -50,7 +48,7 @@ describe("CLI workbench", () => { ["findings", "--json"], capture().stream, capture().stream, - dependencies({ onWorkbench: () => ({ scans: [] }) }), + dependencies({ onWorkbench: () => ({ findings: [] }) }), ), ).toBe(0); for (const confirmed of [[true, false], []]) { @@ -78,55 +76,32 @@ describe("CLI workbench", () => { } }); - test.each([ - { requestedScan: false, selectedTarget: "linked" }, - { requestedScan: true, selectedTarget: "requested" }, - ])( - "lists findings through one trusted target when requested scan exists: $requestedScan", - async ({ requestedScan, selectedTarget }) => { - const repository = resolve("/current/repository"); - const linked = resolve("/current/repository-worktree"); - const findings: JsonObject[] = [ - { occurrenceId: "requested-finding", status: "open" }, - { occurrenceId: "linked-finding", status: "open" }, - ]; - const calls: Array = []; - const stdout = capture(); - expect( - await main( - ["findings", "list", "--json"], - stdout.stream, - capture().stream, - dependencies({ - onWorkbench: (args): JsonObject => { - calls.push(args); - return args[0] === "list-scans" - ? { - scans: [ - { targetId: "linked", targetPath: linked }, - ...(requestedScan - ? [{ targetId: "requested", targetPath: repository }] - : []), - ], - } - : { findings, nextOffset: null }; - }, - }), - ), - ).toBe(0); - expect(calls).toEqual([ - ["list-scans", "--repository", repository], - [ - "list-global-findings", - "--target-id", - selectedTarget, - "--status", - "open", - ], - ]); - expect(JSON.parse(stdout.text())).toEqual({ repository, findings }); - }, - ); + test("preserves the original findings requester instead of selecting a historical alias", async () => { + const repository = resolve("/current/repository"); + const findings: JsonObject[] = [ + { occurrenceId: "requested-finding", status: "open" }, + { occurrenceId: "linked-finding", status: "open" }, + ]; + const calls: Array = []; + const stdout = capture(); + expect( + await main( + ["findings", "list", "--json"], + stdout.stream, + capture().stream, + dependencies({ + onWorkbench: (args): JsonObject => { + calls.push(args); + return { findings, nextOffset: null }; + }, + }), + ), + ).toBe(0); + expect(calls).toEqual([ + ["list-global-findings", "--repository", repository, "--status", "open"], + ]); + expect(JSON.parse(stdout.text())).toEqual({ repository, findings }); + }); test("returns no findings when no related repository has been scanned", async () => { const calls: Array = []; @@ -140,12 +115,14 @@ describe("CLI workbench", () => { dependencies({ onWorkbench: (args) => { calls.push(args); - return { scans: [] }; + return { findings: [], nextOffset: null }; }, }), ), ).toBe(0); - expect(calls).toEqual([["list-scans", "--repository", repository]]); + expect(calls).toEqual([ + ["list-global-findings", "--repository", repository, "--status", "open"], + ]); expect(JSON.parse(stdout.text())).toEqual({ repository, findings: [] }); }); diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts index 09109dbeb..7aab159f6 100644 --- a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -6,13 +6,14 @@ import { PLUGIN_ROOT } from "./plugin-root.js"; const probe = String.raw` import argparse import hashlib +import io import json import os import sqlite3 import stat import sys from collections import Counter -from contextlib import ExitStack +from contextlib import ExitStack, redirect_stderr from pathlib import Path from types import SimpleNamespace from unittest.mock import patch @@ -32,12 +33,16 @@ root = Path.cwd() / "synthetic-identity-fixture" connection = sqlite3.connect(":memory:") connection.row_factory = sqlite3.Row connection.execute("PRAGMA foreign_keys = ON") -identity_migration = next(item for item in MIGRATIONS if item[0] == 31) +legacy_identity_migration = ( + 30, "persist repository identities", + "ALTER TABLE security_targets ADD COLUMN repository_identity TEXT;\n" + "CREATE INDEX security_targets_by_repository_identity ON security_targets(repository_identity);\n", +) initial = ( - (*tuple(item for item in MIGRATIONS if item[0] <= 28), (30, *identity_migration[1:])) + (*tuple(item for item in MIGRATIONS if item[0] <= 28), legacy_identity_migration) if scenario in ("migration", "v30-current") else tuple(item for item in MIGRATIONS if item[0] <= 30) - if scenario == "null-history" else MIGRATIONS + if scenario in ("null-history", "completion-order") else MIGRATIONS ) apply_migrations(connection, initial, lambda: timestamp, lambda database: None) paths = {} @@ -286,6 +291,70 @@ with ExitStack() as stack: "SELECT repository_identity FROM security_targets WHERE id = 'changed'" ).fetchone()[0], })) + elif scenario == "completion-order": + add_target("first", "repository-current") + add_target("second", "repository-current") + add_scan("legacy-b", "second", started="2026-07-31T01:00:00Z", created="2026-07-31T05:00:00+01:00") + add_scan("legacy-a", "first", started="2026-07-31T02:00:00Z", created="2026-07-31T04:00:00Z") + add_scan("legacy-missing", "first", started="2026-07-31T03:00:00Z") + connection.execute("UPDATE scans SET completed_at = NULL WHERE id = 'legacy-missing'") + add_scan("visible-last", "first", started="2026-08-01T01:00:00Z") + add_scan("visible-first", "second", started="2026-08-01T02:00:00Z") + connection.execute( + "UPDATE scans SET status = 'running', completed_at = NULL " + "WHERE id IN ('visible-last', 'visible-first')" + ) + apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) + def sequences(): + return {row["id"]: row["completion_sequence"] for row in connection.execute( + "SELECT id, completion_sequence FROM scans ORDER BY completion_sequence, id" + )} + def predecessors(scan_id): + plan = history.list_unmatched_scan_pairs( + connection, + argparse.Namespace(repository=paths["first"], force=False, after_scan_id=scan_id), + backfill_finding_details=lambda *_: None, read_coverage=lambda _: {}, + ) + return [scan["scanId"] for batch in plan["batches"] for scan in batch["beforeScans"]] + legacy = sequences() + connection.execute( + "UPDATE scans SET status = 'complete', completed_at = '2026-08-01T04:00:00Z' " + "WHERE id = 'visible-first'" + ) + connection.commit() + first_predecessors = predecessors("visible-first") + connection.execute( + "UPDATE scans SET status = 'complete', completed_at = '2026-08-01T03:00:00Z' " + "WHERE id = 'visible-last'" + ) + connection.commit() + add_finding("visible-first", "first-finding") + add_finding("visible-last", "last-finding") + confirmed = {row["findingId"]: row["confirmedInLatestScan"] for row in findings("first")} + last_predecessors = predecessors("visible-last") + reciprocal_predecessors = predecessors("visible-first") + connection.execute("UPDATE scans SET status = 'complete' WHERE id = 'visible-last'") + connection.execute("UPDATE scans SET status = 'failed' WHERE id = 'visible-last'") + add_scan("inserted-complete", "second", created="2026-07-01T00:00:00Z") + before_repair = sequences() + apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) + print(json.dumps({ + "legacy": legacy, + "firstPredecessors": first_predecessors, + "lastPredecessors": last_predecessors, + "reciprocalPredecessors": reciprocal_predecessors, + "confirmed": confirmed, + "sequences": sequences(), + "idempotent": sequences() == before_repair, + "sequenceOutranksFallback": history._scan_completion_order({ + "id": "fallback", "started_at": "2999-01-01T00:00:00Z" + }) < history._scan_completion_order({ + "id": "sequenced", "completion_sequence": 1 + }), + "sealedTimes": {row["id"]: row["completed_at"] for row in connection.execute( + "SELECT id, completed_at FROM scans WHERE id IN ('visible-first', 'visible-last')" + )}, + })) elif scenario == "persisted-alias": for name, stored, live in [ ("requested", "repository-current", None), @@ -296,6 +365,8 @@ with ExitStack() as stack: ]: add_target(name, stored, live) add_scan(name + "-scan", name, "missing" if name == "unverified" else "current") + add_target("fresh", "repository-current") + connection.execute("DELETE FROM security_targets WHERE id = 'fresh'") metadata[paths["reused"]].st_ino += 1000 add_finding("requested-scan", "current-finding") add_finding("reused-scan", "historical-finding", closed=True) @@ -308,11 +379,34 @@ with ExitStack() as stack: feedback = get_scan_feedback( connection, connection.execute("SELECT * FROM scans WHERE id = 'requested-scan'").fetchone() ) + def requested_findings(name): + return indexes.list_global_findings( + connection, + argparse.Namespace(repository=paths[name], target_id=None, limit=50, offset=0, + query=None, severity=None, status=None), + )["findings"] + target_count = connection.execute("SELECT COUNT(*) FROM security_targets").fetchone()[0] + fresh_findings = requested_findings("fresh") + with patch.object(sys, "argv", [ + "workbench", "list-global-findings", "--repository", paths["fresh"], + "--target-id", "reused", + ]), redirect_stderr(io.StringIO()): + try: + workbench.parse_args("Synthetic selector test") + except SystemExit as error: + rejects_both_selectors = error.code == 2 + else: + rejects_both_selectors = False print(json.dumps({ "scans": scans, "replacementRequest": listed("reused"), "aliases": sorted(indexes.repository_target_ids(connection, "requested")), "findings": sorted(row["findingId"] for row in findings("requested")), + "freshFindings": sorted(row["findingId"] for row in fresh_findings), + "replacementFindings": requested_findings("reused"), + "rejectsBothSelectors": rejects_both_selectors, + "readOnly": target_count == connection.execute("SELECT COUNT(*) FROM security_targets").fetchone()[0] + and connection.execute("SELECT 1 FROM security_targets WHERE id = 'fresh'").fetchone() is None, "feedback": [row["findingId"] for row in feedback["falsePositives"]], "matchingCount": matching["scanCount"], "reusedListingProbes": reused_listing_probes, @@ -320,6 +414,60 @@ with ExitStack() as stack: "SELECT repository_identity FROM security_targets WHERE id = 'reused'" ).fetchone()[0], })) + elif scenario == "saved-start": + original_identity = add_target("requested", "repository-current") + other_identity = add_target("other", "repository-other") + add_scan("historical", "requested") + target = Path(paths["requested"]) + target_root = root / "scan-output" + original_mkdir = Path.mkdir + stack.enter_context(patch.object(Path, "mkdir", lambda path, *args, **kwargs: None if path == target_root else original_mkdir(path, *args, **kwargs))) + stack.enter_context(patch.object(workbench, "require_uuid", side_effect=lambda value, label: value)) + stack.enter_context(patch.object(workbench, "require_target", return_value=target)) + stack.enter_context(patch.object(workbench, "require_remediation_target", return_value=target)) + stack.enter_context(patch.object(workbench, "require_scannable_target")) + stack.enter_context(patch.object(workbench, "require_scope", return_value=".")) + stack.enter_context(patch.object(workbench, "directory_snapshot_regular_file_count", return_value=0)) + stack.enter_context(patch.object(workbench, "scan_target_identity", return_value=("synthetic", None, 7, metadata[str(target)].st_ino))) + stack.enter_context(patch.object(workbench, "scan_target_root", return_value=target_root)) + checks = [] + original_ensure = workbench.ensure_security_target + def ensure(database, target_path): + checks.append({"transaction": database.in_transaction, "path": target_path}) + return original_ensure(database, target_path) + stack.enter_context(patch.object(workbench, "ensure_security_target", side_effect=ensure)) + class StartAccepted(Exception): + pass + stack.enter_context(patch.object(workbench, "insert_running_scan", side_effect=StartAccepted)) + accepted = {} + for name, saved_target, live in [ + ("valid", "requested", original_identity), + ("changed-generation", "requested", other_identity), + ("changed-saved-id", "other", original_identity), + ]: + details[str(target)] = live + connection.execute( + "INSERT INTO workspaces (id, target_id, target_path, submitted, created_at, updated_at) " + "VALUES (?, ?, ?, 1, ?, ?)", + (name, saved_target, str(target), timestamp, timestamp), + ) + connection.commit() + try: + workbench.start_scan( + connection, argparse.Namespace(workspace_id=name, scan_root=None, model=None, reasoning_effort=None) + ) + except StartAccepted: + accepted[name] = True + except SystemExit: + accepted[name] = False + print(json.dumps({ + "accepted": accepted, + "verifiedInsideTransaction": len(checks) == 3 and all( + item == {"transaction": True, "path": str(target)} for item in checks + ), + "scanCount": connection.execute("SELECT COUNT(*) FROM scans").fetchone()[0], + "stored": connection.execute("SELECT repository_identity FROM security_targets WHERE id = 'requested'").fetchone()[0], + })) elif scenario == "v30-current": generation_birth = state._timestamp_ns("2026-08-02T00:00:00Z") later = "2026-08-03T00:00:00Z" @@ -341,7 +489,15 @@ with ExitStack() as stack: (name + "-scan", "findings", str(root / name / "findings.json"), timestamp), ) tables = ("scans", "findings", "finding_occurrences", "finding_triage", "scan_artifacts") - retained = {table: [tuple(row) for row in connection.execute("SELECT * FROM " + table)] for table in tables} + columns = { + table: ", ".join(row["name"] for row in connection.execute("PRAGMA table_info(" + table + ")")) + for table in tables + } + def retained_records(): + return {table: [tuple(row) for row in connection.execute( + "SELECT " + columns[table] + " FROM " + table + )] for table in tables} + retained = retained_records() target_ids = sorted(paths) apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) state.backfill_repository_identities(connection) @@ -355,7 +511,7 @@ with ExitStack() as stack: "stored": {row["id"]: row["repository_identity"] for row in connection.execute( "SELECT id, repository_identity FROM security_targets" )}, - "recordsPreserved": retained == {table: [tuple(row) for row in connection.execute("SELECT * FROM " + table)] for table in tables}, + "recordsPreserved": retained == retained_records(), "targetIdsPreserved": target_ids == sorted(row["id"] for row in connection.execute("SELECT id FROM security_targets")), "aliases": sorted(indexes.repository_target_ids(connection, "current-owner")), "removedExact": listed("removed-old"), @@ -532,6 +688,39 @@ test("reuses verified legacy aliases and probes each saved target once per reque expect(result["changedStored"]).toBe("repository-previous"); }); +test("orders completed history by database visibility across legacy and current writers", () => { + const result = run("completion-order"); + + expect(result["legacy"]).toEqual({ + "visible-first": null, + "visible-last": null, + "legacy-missing": 1, + "legacy-a": 2, + "legacy-b": 3, + }); + expect(result["firstPredecessors"]).not.toContain("visible-last"); + expect(result["lastPredecessors"]).toContain("visible-first"); + expect(result["reciprocalPredecessors"]).not.toContain("visible-last"); + expect(result["confirmed"]).toEqual({ + "first-finding": false, + "last-finding": true, + }); + expect(result["sequences"]).toEqual({ + "legacy-missing": 1, + "legacy-a": 2, + "legacy-b": 3, + "visible-first": 4, + "visible-last": 5, + "inserted-complete": 6, + }); + expect(result["idempotent"]).toBe(true); + expect(result["sequenceOutranksFallback"]).toBe(true); + expect(result["sealedTimes"]).toEqual({ + "visible-first": "2026-08-01T04:00:00Z", + "visible-last": "2026-08-01T03:00:00Z", + }); +}); + test("keeps authenticated historical aliases visible without trusting a replacement checkout", () => { const result = run("persisted-alias"); @@ -543,6 +732,13 @@ test("keeps authenticated historical aliases visible without trusting a replacem expect(result["replacementRequest"]).toEqual([]); expect(result["aliases"]).toEqual(["legacy", "requested", "reused"]); expect(result["findings"]).toEqual(["current-finding", "historical-finding"]); + expect(result["freshFindings"]).toEqual([ + "current-finding", + "historical-finding", + ]); + expect(result["replacementFindings"]).toEqual([]); + expect(result["rejectsBothSelectors"]).toBe(true); + expect(result["readOnly"]).toBe(true); expect(result["feedback"]).toEqual(["historical-finding"]); expect(result["matchingCount"]).toBe(3); expect(result["reusedListingProbes"]).toBe(0); @@ -592,6 +788,19 @@ test("admits rerun lineage only through the verified repository and exact scope" expect(result["scanCount"]).toBe(7); }); +test("revalidates saved workspace identity inside the scan-start transaction", () => { + const result = run("saved-start"); + + expect(result["accepted"]).toEqual({ + valid: true, + "changed-generation": false, + "changed-saved-id": false, + }); + expect(result["verifiedInsideTransaction"]).toBe(true); + expect(result["scanCount"]).toBe(1); + expect(result["stored"]).toBe("repository-current"); +}); + test("quarantines unproved public-v30 bindings without discarding historical records", () => { const result = run("v30-current"); diff --git a/sdk/typescript/tests-ts/workbench-target-migration.test.ts b/sdk/typescript/tests-ts/workbench-target-migration.test.ts index 03a26c1c6..ab9139154 100644 --- a/sdk/typescript/tests-ts/workbench-target-migration.test.ts +++ b/sdk/typescript/tests-ts/workbench-target-migration.test.ts @@ -155,6 +155,11 @@ identity_migration = next( migration for migration in MIGRATIONS if migration[1] == "persist repository identities" ) identity_version = identity_migration[0] +legacy_identity_migration = ( + 30, "persist repository identities", + "ALTER TABLE security_targets ADD COLUMN repository_identity TEXT;\n" + "CREATE INDEX security_targets_by_repository_identity ON security_targets(repository_identity);\n", +) historical = tuple(migration for migration in MIGRATIONS if migration[0] <= 28) published = tuple(migration for migration in MIGRATIONS if migration[0] < identity_version) apply_migrations( @@ -193,7 +198,7 @@ if scenario == "out-of-order-publication-migrations": elif scenario == "pre-release-identity-version": apply_migrations( connection, - (*historical, (30, *identity_migration[1:])), + (*historical, legacy_identity_migration), lambda: timestamp, backfill, ) @@ -211,6 +216,12 @@ indexes = { row["name"]: row for row in connection.execute("PRAGMA index_list(security_targets)") } +scan_columns = { + row["name"]: row for row in connection.execute("PRAGMA table_info(scans)") +} +scan_indexes = { + row["name"]: row for row in connection.execute("PRAGMA index_list(scans)") +} print(json.dumps({ "backfillCalls": len(backfill_calls), "hasRepositoryIdentityColumn": "repository_identity" in columns, @@ -233,6 +244,16 @@ print(json.dumps({ "repositoryIdentityIndexIsUnique": bool( indexes["security_targets_by_repository_identity"]["unique"] ), + "completionSequenceIsNullable": not bool( + scan_columns["completion_sequence"]["notnull"] + ), + "completionSequenceIndexIsUnique": bool( + scan_indexes["scans_completion_sequence"]["unique"] + ), + "completionSequenceTriggers": sorted(row["name"] for row in connection.execute( + "SELECT name FROM sqlite_master WHERE type = 'trigger' " + "AND name LIKE 'scans_assign_%_completion_sequence'" + )), "migrationName": connection.execute( "SELECT name FROM schema_migrations WHERE version = ?", (identity_version,) ).fetchone()[0], @@ -381,6 +402,9 @@ describe("stable workbench target migration", () => { publicationMigrations: Record; repositoryIdentityColumnIsNullable: boolean; repositoryIdentityIndexIsUnique: boolean; + completionSequenceIsNullable: boolean; + completionSequenceIndexIsUnique: boolean; + completionSequenceTriggers: string[]; targetIdentity: string | null; targetId: string; teamOnlyPublicationIndexes: string[]; @@ -397,6 +421,12 @@ describe("stable workbench target migration", () => { }, repositoryIdentityColumnIsNullable: true, repositoryIdentityIndexIsUnique: false, + completionSequenceIsNullable: true, + completionSequenceIndexIsUnique: true, + completionSequenceTriggers: [ + "scans_assign_inserted_completion_sequence", + "scans_assign_updated_completion_sequence", + ], targetIdentity: null, targetId: "target-existing", teamOnlyPublicationIndexes: [ From 337abb644a319135d7c4be4a5fe26a0912ce10a1 Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sun, 16 Aug 2026 05:03:52 -0700 Subject: [PATCH 12/21] fix(workbench): preserve findings and Linux identities --- .../scripts/workbench_native_indexes.py | 11 +- .../scripts/workbench_target_state.py | 66 +++++++ sdk/typescript/src/api.ts | 7 +- sdk/typescript/tests-ts/api.test.ts | 40 +++- sdk/typescript/tests-ts/cli-workbench.test.ts | 14 ++ .../tests-ts/workbench-identity-cache.test.ts | 171 +++++++++++++++++- .../workbench-repository-identity.test.ts | 3 +- 7 files changed, 296 insertions(+), 16 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py index e71b9fc5c..a56166dca 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py @@ -34,14 +34,18 @@ def list_global_findings( query = args.query.strip().casefold() if args.query else "" identities = RepositoryIdentityCache(connection) repository = getattr(args, "repository", None) + requested = None if repository is not None: - target_ids = identities.target_ids_for_path( - str(Path(repository).expanduser().resolve()) - ) + repository = str(Path(repository).expanduser().resolve()) + requested = identities.for_path(repository) + target_ids = identities.target_ids_for_path(repository) else: target_ids = ( None if args.target_id is None else identities.target_ids(args.target_id) ) + target = identities.targets.get(args.target_id) + if identities.supports_identity and target is not None: + requested = identities.for_row(target) findings = ( row for row in _indexed_findings(connection, identities=identities, target_ids=target_ids) @@ -90,6 +94,7 @@ def list_global_findings( "limit": limit, "nextOffset": args.offset + limit if has_more else None, "offset": args.offset, + "projectionAvailable": requested is None or requested.ownership_matches, } diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index 6bd632d9e..e20a88da6 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -3,8 +3,10 @@ from __future__ import annotations import argparse +import ctypes import hashlib import os +import platform import sqlite3 import stat import subprocess @@ -46,6 +48,67 @@ def _repository_worktree(target: Path) -> tuple[Path, str] | None: return canonical_root, relative.as_posix() +class _LinuxStatxTimestamp(ctypes.Structure): + _fields_ = ( + ("seconds", ctypes.c_int64), + ("nanoseconds", ctypes.c_uint32), + ("_reserved", ctypes.c_int32), + ) + + +class _LinuxStatx(ctypes.Structure): + # Linux's statx UAPI keeps birth time at offset 0x50 in a 0x100-byte buffer. + _fields_ = ( + ("mask", ctypes.c_uint32), + ("_before_birth_time", ctypes.c_ubyte * 76), + ("birth_time", _LinuxStatxTimestamp), + ("_after_birth_time", ctypes.c_ubyte * 160), + ) + + +def _linux_repository_birth_time_ns(path: str) -> int | None: + if ctypes.sizeof(_LinuxStatx) != 256 or _LinuxStatx.birth_time.offset != 80: + return None + metadata = _LinuxStatx() + birth_time_mask = 0x800 + arguments = (-100, os.fsencode(path), 0, birth_time_mask, ctypes.byref(metadata)) + try: + libc = ctypes.CDLL(None, use_errno=True) + try: + statx = libc.statx + except AttributeError: + # Older musl has the syscall but no statx wrapper. Only use known LP64 ABIs. + if ctypes.sizeof(ctypes.c_void_p) != 8 or ctypes.sizeof(ctypes.c_long) != 8: + return None + number = {"x86_64": 332, "aarch64": 291}.get(platform.machine()) + if number is None: + return None + statx = libc.syscall + statx.argtypes = ( + ctypes.c_long, ctypes.c_long, ctypes.c_char_p, ctypes.c_long, + ctypes.c_ulong, ctypes.POINTER(_LinuxStatx), + ) + statx.restype = ctypes.c_long + status = statx(number, *arguments) + else: + statx.argtypes = ( + ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_uint, + ctypes.POINTER(_LinuxStatx), + ) + statx.restype = ctypes.c_int + status = statx(*arguments) + except (AttributeError, OSError): + return None + birth_time = metadata.birth_time + if ( + status != 0 or not metadata.mask & birth_time_mask + or not 0 <= birth_time.nanoseconds < 1_000_000_000 + ): + return None + birth_time_ns = birth_time.seconds * 1_000_000_000 + birth_time.nanoseconds + return birth_time_ns if birth_time_ns > 0 else None + + def _repository_birth_time_ns(path: str, metadata: os.stat_result) -> int | None: birth_time_ns = getattr(metadata, "st_birthtime_ns", None) if birth_time_ns is not None: @@ -57,6 +120,9 @@ def _repository_birth_time_ns(path: str, metadata: os.stat_result) -> int | None return int(birth_time * 1_000_000_000) if birth_time > 0 else None if not sys.platform.startswith("linux"): return None + birth_time_ns = _linux_repository_birth_time_ns(path) + if birth_time_ns is not None: + return birth_time_ns try: result = subprocess.run( ["stat", "--format=%.9W", "--", path], diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts index d29f3845c..d17d83a71 100644 --- a/sdk/typescript/src/api.ts +++ b/sdk/typescript/src/api.ts @@ -1939,7 +1939,12 @@ export async function listRepositoryFindings( ...(status === "open" ? ["--status", "open"] : []), ...(offset === undefined ? [] : ["--offset", String(offset)]), ]); - if (!Array.isArray(page["findings"])) return undefined; + if ( + page["projectionAvailable"] === false || + !Array.isArray(page["findings"]) + ) { + return undefined; + } findings.push(...(page["findings"] as JsonObject[])); offset = typeof page["nextOffset"] === "number" ? page["nextOffset"] : undefined; diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts index 37896482a..a2fc5596e 100644 --- a/sdk/typescript/tests-ts/api.test.ts +++ b/sdk/typescript/tests-ts/api.test.ts @@ -38,6 +38,7 @@ import { import { classifyConnectionFailure, initialCredentialsAvailable, + listRepositoryFindings, } from "../src/api.js"; import { FIREWORKS_CODEX_PROVIDER, @@ -2903,9 +2904,38 @@ describe("CodexSecurity orchestration", () => { }, ); + test("discards partial findings when a repository projection becomes unavailable", async () => { + const pages: JsonObject[] = [ + { findings: [{ findingId: "first-page" }], nextOffset: 1 }, + { findings: [], projectionAvailable: false, nextOffset: null }, + ]; + const commands: Array = []; + expect( + await listRepositoryFindings(async (args) => { + commands.push(args); + return pages[commands.length - 1]!; + }, "target_sha256_example"), + ).toBeUndefined(); + expect(commands).toHaveLength(2); + expect(commands[1]).toEqual([...commands[0]!, "--offset", "1"]); + expect( + await listRepositoryFindings( + async () => ({ findings: [], projectionAvailable: true }), + "target_sha256_example", + ), + ).toEqual([]); + expect( + await listRepositoryFindings( + async () => ({ findings: [] }), + "target_sha256_example", + ), + ).toEqual([]); + }); + test.each([ ["semantic matching fails", "matcher", "matcher unavailable"], ["the repository index fails", "index", "index unavailable"], + ["the repository projection is unavailable", "projection", undefined], ["a cost limit still allows false-positive matching", "budget", undefined], [ "dismissed history survives missing reviewer feedback", @@ -2971,6 +3001,9 @@ describe("CodexSecurity orchestration", () => { } if (args[0] === "list-global-findings") { if (failure === "index") throw new Error("index unavailable"); + if (failure === "projection") { + return { findings: [], projectionAvailable: false }; + } if (failure === "dismissed") { return { findings: args.includes("--status") @@ -3036,7 +3069,12 @@ describe("CodexSecurity orchestration", () => { ? [] : [`Could not update repository findings: ${warning}`], ); - expect(modelCalled).toBe(failure !== "index"); + expect(modelCalled).toBe(failure !== "index" && failure !== "projection"); + if (failure === "projection") { + expect( + result.findings.findings.map(({ findingId }) => findingId), + ).toContain(current.findingId); + } expect(commands.some(([command]) => command === "complete-scan")).toBe( true, ); diff --git a/sdk/typescript/tests-ts/cli-workbench.test.ts b/sdk/typescript/tests-ts/cli-workbench.test.ts index 1fa7c7e3f..19f6461ab 100644 --- a/sdk/typescript/tests-ts/cli-workbench.test.ts +++ b/sdk/typescript/tests-ts/cli-workbench.test.ts @@ -76,6 +76,20 @@ describe("CLI workbench", () => { } }); + test("uses sealed scan findings when the repository projection is unavailable", async () => { + const stderr = capture(); + expect( + await main( + ["scan"], + capture().stream, + stderr.stream, + dependencies({ result: fakeResult(["high"]) }), + ), + ).toBe(0); + expect(stderr.text()).toMatch(/FINDINGS\s+1\b/u); + expect(stderr.text()).toContain("1 high"); + }); + test("preserves the original findings requester instead of selecting a historical alias", async () => { const repository = resolve("/current/repository"); const findings: JsonObject[] = [ diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts index 7aab159f6..b19420112 100644 --- a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -5,6 +5,7 @@ import { PLUGIN_ROOT } from "./plugin-root.js"; const probe = String.raw` import argparse +import ctypes import hashlib import io import json @@ -16,7 +17,7 @@ from collections import Counter from contextlib import ExitStack, redirect_stderr from pathlib import Path from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import Mock, patch sys.path.insert(0, sys.argv[1]) import workbench_scan_history as history @@ -202,7 +203,98 @@ with ExitStack() as stack: stack.enter_context(patch.object(Path, "lstat", lstat_path)) stack.enter_context(patch.object(state, "_repository_identity_details", identity_details)) stack.enter_context(patch.object(state, "repository_origin", origin)) - if scenario == "cache": + if scenario == "birth-time": + path = "/synthetic repository/.git" + real_sizeof = ctypes.sizeof + + def native_result(*, wrapper=True, machine="x86_64", mask=0x800, + seconds=42, nanoseconds=123, status=0, + pointer_size=8, long_size=8): + calls = [] + + def invoke(*arguments): + number = None + if not wrapper: + number, *arguments = arguments + directory, encoded, flags, requested_mask, output = arguments + assert (directory, encoded, flags, requested_mask) == ( + -100, os.fsencode(path), 0, 0x800 + ) + value = ctypes.cast(output, ctypes.POINTER(state._LinuxStatx)).contents + assert not value.mask and not value.birth_time.seconds + value.mask = mask + value.birth_time.seconds = seconds + value.birth_time.nanoseconds = nanoseconds + calls.append(number) + return status + + function = Mock(side_effect=invoke) + libc = SimpleNamespace(**{"statx" if wrapper else "syscall": function}) + + def sizeof(kind): + if kind is ctypes.c_void_p: + return pointer_size + if kind is ctypes.c_long: + return long_size + return real_sizeof(kind) + + with patch.object(state.ctypes, "CDLL", return_value=libc), \ + patch.object(state.ctypes, "sizeof", side_effect=sizeof), \ + patch.object(state.platform, "machine", return_value=machine): + result = state._linux_repository_birth_time_ns(path) + expected_arguments = ( + (ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_uint, + ctypes.POINTER(state._LinuxStatx)) if wrapper else + (ctypes.c_long, ctypes.c_long, ctypes.c_char_p, ctypes.c_long, + ctypes.c_ulong, ctypes.POINTER(state._LinuxStatx)) + ) + return { + "value": result, "calls": calls, + "typed": not calls or function.argtypes == expected_arguments + and function.restype is (ctypes.c_int if wrapper else ctypes.c_long), + } + + cases = { + "wrapper": native_result(), + "x86_64": native_result(wrapper=False), + "aarch64": native_result(wrapper=False, machine="aarch64"), + "unknown-abi": native_result(wrapper=False, machine="unknown"), + "pointer32": native_result(wrapper=False, pointer_size=4), + "long32": native_result(wrapper=False, long_size=4), + "missing-mask": native_result(mask=0), + "invalid-nanoseconds": native_result(nanoseconds=1_000_000_000), + "zero": native_result(seconds=0, nanoseconds=0), + "negative": native_result(seconds=-1, nanoseconds=999_999_999), + "failed": native_result(status=-1), + } + with patch.object(state.ctypes, "CDLL", return_value=SimpleNamespace()), \ + patch.object(state.platform, "machine", return_value="x86_64"): + cases["missing-symbols"] = state._linux_repository_birth_time_ns(path) + with patch.object(state.ctypes, "CDLL", side_effect=OSError("unavailable")): + cases["missing-libc"] = state._linux_repository_birth_time_ns(path) + with patch.object(state.ctypes, "sizeof", return_value=255), \ + patch.object(state.ctypes, "CDLL") as library: + cases["invalid-layout"] = state._linux_repository_birth_time_ns(path) + assert not library.called + with patch.object(sys, "platform", "linux"), patch.object(os, "name", "posix"), \ + patch.object(state, "_linux_repository_birth_time_ns", return_value=42_000_000_123), \ + patch.object(state.subprocess, "run") as command: + native = state._repository_birth_time_ns(path, SimpleNamespace(st_ctime_ns=41)) + assert not command.called + with patch.object(sys, "platform", "linux"), patch.object(os, "name", "posix"), \ + patch.object(state, "_linux_repository_birth_time_ns", return_value=None), \ + patch.object(state.subprocess, "run", return_value=SimpleNamespace( + stdout="42.000000123\n", returncode=0 + )) as command: + fallback = state._repository_birth_time_ns(path, SimpleNamespace(st_ctime_ns=41)) + assert command.call_args.args[0] == ["stat", "--format=%.9W", "--", path] + assert command.call_args.kwargs["env"]["LC_ALL"] == "C" + print(json.dumps({ + "cases": cases, "native": native, "fallback": fallback, + "layout": [real_sizeof(state._LinuxStatxTimestamp), + state._LinuxStatx.birth_time.offset, real_sizeof(state._LinuxStatx)], + })) + elif scenario == "cache": for name, stored, live in [ ("requested", "repository-current", None), ("persisted", "repository-current", None), @@ -365,8 +457,9 @@ with ExitStack() as stack: ]: add_target(name, stored, live) add_scan(name + "-scan", name, "missing" if name == "unverified" else "current") - add_target("fresh", "repository-current") - connection.execute("DELETE FROM security_targets WHERE id = 'fresh'") + for name, identity in [("fresh", "repository-current"), ("empty", "repository-empty")]: + add_target(name, identity) + connection.execute("DELETE FROM security_targets WHERE id = ?", (name,)) metadata[paths["reused"]].st_ino += 1000 add_finding("requested-scan", "current-finding") add_finding("reused-scan", "historical-finding", closed=True) @@ -379,14 +472,17 @@ with ExitStack() as stack: feedback = get_scan_feedback( connection, connection.execute("SELECT * FROM scans WHERE id = 'requested-scan'").fetchone() ) - def requested_findings(name): + def findings_page(name=None, target_id=None): return indexes.list_global_findings( connection, - argparse.Namespace(repository=paths[name], target_id=None, limit=50, offset=0, + argparse.Namespace(repository=paths[name] if name else None, + target_id=target_id, limit=50, offset=0, query=None, severity=None, status=None), - )["findings"] + ) target_count = connection.execute("SELECT COUNT(*) FROM security_targets").fetchone()[0] - fresh_findings = requested_findings("fresh") + fresh_findings = findings_page("fresh") + empty_findings = findings_page("empty") + replacement_findings = findings_page("reused") with patch.object(sys, "argv", [ "workbench", "list-global-findings", "--repository", paths["fresh"], "--target-id", "reused", @@ -402,8 +498,16 @@ with ExitStack() as stack: "replacementRequest": listed("reused"), "aliases": sorted(indexes.repository_target_ids(connection, "requested")), "findings": sorted(row["findingId"] for row in findings("requested")), - "freshFindings": sorted(row["findingId"] for row in fresh_findings), - "replacementFindings": requested_findings("reused"), + "freshFindings": sorted(row["findingId"] for row in fresh_findings["findings"]), + "emptyFindings": empty_findings["findings"], + "replacementFindings": replacement_findings["findings"], + "projectionAvailable": { + "fresh": fresh_findings["projectionAvailable"], + "empty": empty_findings["projectionAvailable"], + "replacementPath": replacement_findings["projectionAvailable"], + "replacementId": findings_page(target_id="reused")["projectionAvailable"], + "unknownId": findings_page(target_id="not-registered")["projectionAvailable"], + }, "rejectsBothSelectors": rejects_both_selectors, "readOnly": target_count == connection.execute("SELECT COUNT(*) FROM security_targets").fetchone()[0] and connection.execute("SELECT 1 FROM security_targets WHERE id = 'fresh'").fetchone() is None, @@ -647,6 +751,45 @@ function run(scenario: string): Record { return JSON.parse(execution.stdout) as Record; } +test("reads precise Linux birth time through supported native interfaces", () => { + const result = run("birth-time"); + const cases = result["cases"] as Record; + + expect(result["layout"]).toEqual([16, 80, 256]); + expect(result["native"]).toBe(42_000_000_123); + expect(result["fallback"]).toBe(result["native"]); + expect(cases["wrapper"]).toEqual({ + value: 42_000_000_123, + calls: [null], + typed: true, + }); + for (const [architecture, number] of [ + ["x86_64", 332], + ["aarch64", 291], + ] as const) { + expect(cases[architecture]).toEqual({ + value: 42_000_000_123, + calls: [number], + typed: true, + }); + } + for (const name of ["unknown-abi", "pointer32", "long32"]) { + expect(cases[name]).toEqual({ value: null, calls: [], typed: true }); + } + for (const name of [ + "missing-mask", + "invalid-nanoseconds", + "zero", + "negative", + "failed", + ]) { + expect(cases[name]).toEqual({ value: null, calls: [null], typed: true }); + } + for (const name of ["missing-symbols", "missing-libc", "invalid-layout"]) { + expect(cases[name]).toBeNull(); + } +}); + test("reuses verified legacy aliases and probes each saved target once per request", () => { const result = run("cache"); @@ -737,6 +880,14 @@ test("keeps authenticated historical aliases visible without trusting a replacem "historical-finding", ]); expect(result["replacementFindings"]).toEqual([]); + expect(result["emptyFindings"]).toEqual([]); + expect(result["projectionAvailable"]).toEqual({ + fresh: true, + empty: true, + replacementPath: false, + replacementId: false, + unknownId: true, + }); expect(result["rejectsBothSelectors"]).toBe(true); expect(result["readOnly"]).toBe(true); expect(result["feedback"]).toEqual(["historical-finding"]); diff --git a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts index bb2b5e269..5c75f0611 100644 --- a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts +++ b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts @@ -250,7 +250,8 @@ if scenario == "identity": windows_legacy_birth_time = _repository_birth_time_ns(common, legacy_metadata) windows_modern_birth_time = _repository_birth_time_ns(common, modern_metadata) linux_birth_times = {} - with patch.object(sys, "platform", "linux"), patch.object(os, "name", "posix"): + with patch.object(sys, "platform", "linux"), patch.object(os, "name", "posix"), \ + patch("workbench_target_state._linux_repository_birth_time_ns", return_value=None): for label, output, status in ( ("valid", "42.000000123\n", 0), ("unavailable", "0.000000000\n", 0), From 7a9e4b6a1901fb5d08261c1e727cae82c3000049 Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sun, 16 Aug 2026 06:08:21 -0700 Subject: [PATCH 13/21] fix(workbench): preserve unbound legacy history --- sdk/typescript/README.md | 7 +- .../scripts/workbench_scan_history.py | 13 +- .../scripts/workbench_schema.py | 4 + .../scripts/workbench_target_state.py | 125 ++++++---- sdk/typescript/src/cli.ts | 19 +- sdk/typescript/tests-ts/cli-workbench.test.ts | 47 +++- .../tests-ts/workbench-identity-cache.test.ts | 230 +++++++++++++++++- .../workbench-repository-identity.test.ts | 11 +- 8 files changed, 374 insertions(+), 82 deletions(-) diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 8c1861ef0..3834218fd 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -667,8 +667,11 @@ const directPublication = await publishScan("/path/to/completed-scan", { `scans` or `scans list` lists scans for the current repository. Linked Git worktrees are discovered automatically and grouped when they share the selected -state database. Findings indicate whether they were confirmed in the repository's -latest completed scan across those linked worktrees. Finding confirmation and +state database. History written without a recorded Git generation stays attached +to its original target and scan IDs. If an older client creates and scans a +target before its identity can be recorded, later clients do not retroactively +share that history across worktrees. Findings indicate whether they were confirmed +in the repository's latest completed scan across those linked worktrees. Finding confirmation and automatic matching use the order in which completed scans become visible in the workbench; sealed report timestamps are unchanged. Pass a repository path to inspect another checkout. `--scan-root DIR` only filters scans already recorded diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py index 0460c4823..613d4e2c1 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py @@ -29,13 +29,17 @@ def _same_repository( *, identities: RepositoryIdentityCache | None = None, ) -> bool: - identities = identities or RepositoryIdentityCache(connection) before_stored = ( before["repository_identity"] if "repository_identity" in before.keys() else None ) after_stored = ( after["repository_identity"] if "repository_identity" in after.keys() else None ) + if before_stored is not None and before_stored == after_stored: + return True + if before["target_id"] and before["target_id"] == after["target_id"]: + return before_stored is None or after_stored is None + identities = identities or RepositoryIdentityCache(connection) before_state = identities.for_row(before) after_state = identities.for_row(after) if before_state.resolved_path is None or after_state.resolved_path is None: @@ -44,10 +48,7 @@ def _same_repository( after_group = after_stored or after_state.verified_identity if before_group is not None and before_group == after_group: return True - if ( - before_state.target_id and before_state.target_id == after_state.target_id - or before_state.resolved_path == after_state.resolved_path - ): + if before_state.resolved_path == after_state.resolved_path: return before_stored is None or after_stored is None before_identity = before_state.verified_identity after_identity = after_state.verified_identity @@ -258,7 +259,7 @@ def list_unmatched_scan_pairs( repository = Path(args.repository).expanduser().resolve() identities = RepositoryIdentityCache(connection) requested = identities.for_path(str(repository)) - if identities.supports_identity and requested.target_id: + if identities.supports_identity: requested.require_owner() requested_row = { "target_id": requested.target_id, diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py index 4365feafe..ebadf69f4 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py @@ -807,6 +807,10 @@ def apply_migrations( repair_deep_scan_failure_counter_migration(connection) if should_backfill_targets: backfill_security_targets(connection) + elif any(version == 31 for version, _, _ in migrations): + from workbench_target_state import backfill_repository_identities + + backfill_repository_identities(connection) connection.commit() except BaseException: connection.rollback() diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index e20a88da6..76784b68d 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -250,7 +250,8 @@ class RepositoryTargetState: repository: GitRepositoryIdentity | None = None ownership_matches: bool = False strict_owner_matches: bool = False - generation_matches_history: bool = False + generation_predates_history: bool = False + has_historical_scans: bool = False missing: bool = False @property @@ -262,9 +263,12 @@ def verified_identity(self) -> str | None: if not self.ownership_matches or self.repository is None: return None if self.stored_identity is None: + # Checkout ownership and repository age cannot identify an older Git generation. return ( self.live_identity - if self.strict_owner_matches and self.generation_matches_history + if self.strict_owner_matches + and self.generation_predates_history + and not self.has_historical_scans else None ) return self.live_identity if self.live_identity == self.stored_identity else None @@ -320,7 +324,7 @@ def _inspect_repository_target( malformed_owner = False mismatch = False strict_owner_matches = False - generation_matches_history = False + generation_predates_history = False ownership_matches = True if {"target_id", "target_path"} <= scan_columns: if not {"target_device", "target_inode"} <= scan_columns: @@ -329,7 +333,7 @@ def _inspect_repository_target( (target_id, target_path), ).fetchone() is not None strict_owner_matches = not historical_scan - generation_matches_history = repository is not None and not historical_scan + generation_predates_history = repository is not None and not historical_scan else: strict_owner_matches = True timestamps = ", started_at, created_at" if { @@ -342,7 +346,7 @@ def _inspect_repository_target( """, (target_id, target_path), ).fetchall() - generation_matches_history = ( + generation_predates_history = ( repository is not None and _repository_predates_history(repository, scans) ) for scan in scans: @@ -381,11 +385,11 @@ def _inspect_repository_target( or stored_identity is None and repository is not None and historical_scan - and not generation_matches_history + and not generation_predates_history ) return RepositoryTargetState( target_id, target_path, stored_identity, resolved_path, metadata, repository, - ownership_matches, strict_owner_matches, generation_matches_history, + ownership_matches, strict_owner_matches, generation_predates_history, historical_scan, ) @@ -576,14 +580,36 @@ def normalize_pre_release_repository_identities(connection: sqlite3.Connection) ) +def _bind_unscanned_repository_identity( + connection: sqlite3.Connection, target_id: str, target_path: str, identity: str +) -> bool: + return connection.execute( + """ + UPDATE security_targets + SET repository_identity = ? + WHERE id = ? AND current_path = ? AND repository_identity IS NULL + AND NOT EXISTS ( + SELECT 1 FROM scans + WHERE scans.target_id = security_targets.id + OR scans.target_path = security_targets.current_path + ) + """, + (identity, target_id, target_path), + ).rowcount == 1 + + def backfill_repository_identities(connection: sqlite3.Connection) -> None: if not supports_repository_identity(connection): return targets = connection.execute( """ - SELECT id, current_path - FROM security_targets - WHERE repository_identity IS NULL + SELECT targets.id, targets.current_path + FROM security_targets AS targets + WHERE targets.repository_identity IS NULL + AND NOT EXISTS ( + SELECT 1 FROM scans + WHERE scans.target_id = targets.id OR scans.target_path = targets.current_path + ) """ ).fetchall() for target in targets: @@ -591,13 +617,8 @@ def backfill_repository_identities(connection: sqlite3.Connection) -> None: connection, str(target["id"]), target["current_path"] ) if identity is not None: - connection.execute( - """ - UPDATE security_targets - SET repository_identity = ? - WHERE id = ? AND repository_identity IS NULL - """, - (identity, target["id"]), + _bind_unscanned_repository_identity( + connection, str(target["id"]), target["current_path"], identity ) @@ -627,45 +648,16 @@ def ensure_security_target( connection: sqlite3.Connection, target_path: str, *, verify_ownership: bool = True ) -> str: supports_identity = supports_repository_identity(connection) - existing = connection.execute( + target_query = ( "SELECT id, repository_identity FROM security_targets WHERE current_path = ?" if supports_identity - else "SELECT id FROM security_targets WHERE current_path = ?", - (target_path,), - ).fetchone() - if existing is not None: - target_id = str(existing["id"]) - if supports_identity: - state = _inspect_repository_target( - connection, target_id, target_path, existing["repository_identity"] - ) - if verify_ownership: - state.require_owner() - if supports_identity and existing["repository_identity"] is None: - identity = state.verified_identity - if identity is not None: - connection.execute( - """ - UPDATE security_targets - SET repository_identity = ? - WHERE id = ? AND repository_identity IS NULL - """, - (identity, target_id), - ) - return target_id - target_id = stable_target_id(Path(target_path)) - timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - if supports_identity: - identity = verified_repository_identity(connection, target_id, target_path) - connection.execute( - """ - INSERT OR IGNORE INTO security_targets ( - id, current_path, display_name, created_at, updated_at, repository_identity - ) VALUES (?, ?, ?, ?, ?, ?) - """, - (target_id, target_path, Path(target_path).name, timestamp, timestamp, identity), - ) - else: + else "SELECT id FROM security_targets WHERE current_path = ?" + ) + existing = connection.execute(target_query, (target_path,)).fetchone() + if existing is None: + target_id = stable_target_id(Path(target_path)) + timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + # Leave repository_identity NULL until the guarded binding below succeeds. connection.execute( """ INSERT OR IGNORE INTO security_targets ( @@ -674,6 +666,31 @@ def ensure_security_target( """, (target_id, target_path, Path(target_path).name, timestamp, timestamp), ) + existing = connection.execute(target_query, (target_path,)).fetchone() + if existing is None: + raise SystemExit("The repository target changed while it was being registered.") + target_id = str(existing["id"]) + if not supports_identity: + return target_id + state = _inspect_repository_target( + connection, target_id, target_path, existing["repository_identity"] + ) + if verify_ownership: + state.require_owner() + if existing["repository_identity"] is None: + identity = state.verified_identity + if identity is not None and not _bind_unscanned_repository_identity( + connection, target_id, target_path, identity + ): + # A writer may have registered history or bound this target after inspection. + existing = connection.execute(target_query, (target_path,)).fetchone() + if existing is None: + raise SystemExit("The repository target changed while it was being registered.") + target_id = str(existing["id"]) + if verify_ownership: + _inspect_repository_target( + connection, target_id, target_path, existing["repository_identity"] + ).require_owner() return target_id diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 5a63ffdcd..c51c15a63 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -1240,13 +1240,18 @@ export async function main( args.repository ?? ".", ); return presentHistory( - await history(async () => ({ - repository, - findings: - (await listRepositoryFindings(dependencies.runWorkbench, { - repository, - })) ?? [], - })), + await history(async () => { + const findings = await listRepositoryFindings( + dependencies.runWorkbench, + { repository }, + ); + if (findings === undefined) { + throw new CodexSecurityError( + "Repository findings are unavailable for the requested checkout.", + ); + } + return { repository, findings }; + }), "findings", format, { repository }, diff --git a/sdk/typescript/tests-ts/cli-workbench.test.ts b/sdk/typescript/tests-ts/cli-workbench.test.ts index 19f6461ab..4a966d32d 100644 --- a/sdk/typescript/tests-ts/cli-workbench.test.ts +++ b/sdk/typescript/tests-ts/cli-workbench.test.ts @@ -129,7 +129,11 @@ describe("CLI workbench", () => { dependencies({ onWorkbench: (args) => { calls.push(args); - return { findings: [], nextOffset: null }; + return { + findings: [], + nextOffset: null, + projectionAvailable: true, + }; }, }), ), @@ -140,6 +144,47 @@ describe("CLI workbench", () => { expect(JSON.parse(stdout.text())).toEqual({ repository, findings: [] }); }); + test("reports unavailable findings on any page instead of claiming an empty repository", async () => { + for (const laterPage of [false, true]) { + for (const json of [false, true]) { + const stdout = capture(!json); + const stderr = capture(); + const responses: JsonObject[] = [ + ...(laterPage + ? [ + { + findings: [{ title: "Synthetic finding" }], + nextOffset: 1, + projectionAvailable: true, + }, + ] + : []), + { + findings: [], + nextOffset: null, + projectionAvailable: false, + }, + ]; + const calls: Array = []; + expect( + await main( + ["findings", "list", ...(json ? ["--json"] : [])], + stdout.stream, + stderr.stream, + dependencies({ + onWorkbench: (args) => responses[calls.push(args) - 1]!, + }), + ), + ).toBe(2); + expect(calls).toHaveLength(laterPage ? 2 : 1); + expect(stderr.text()).toContain("findings are unavailable"); + expect(stdout.text()).not.toContain("0 open findings"); + expect(stdout.text()).not.toContain("Synthetic finding"); + expect(stdout.text()).not.toContain('"findings": []'); + } + } + }); + test("lists repository and scan-root history without starting Codex", async () => { const repository = resolve("/current/repository"); const cases: Array<[string[], string[]]> = [ diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts index b19420112..d079d54c7 100644 --- a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -41,9 +41,9 @@ legacy_identity_migration = ( ) initial = ( (*tuple(item for item in MIGRATIONS if item[0] <= 28), legacy_identity_migration) - if scenario in ("migration", "v30-current") else + if scenario in ("migration", "v30-current", "completion-order") else tuple(item for item in MIGRATIONS if item[0] <= 30) - if scenario in ("null-history", "completion-order") else MIGRATIONS + if scenario == "null-history" else MIGRATIONS ) apply_migrations(connection, initial, lambda: timestamp, lambda database: None) paths = {} @@ -307,6 +307,7 @@ with ExitStack() as stack: ("changed", "repository-previous", "repository-current"), ]: add_target(name, stored, live) + state.ensure_security_target(connection, paths["legacy"]) for name in paths: add_scan(name + "-scan", name, "missing" if name == "unverified" else "current") for number in range(5): @@ -456,6 +457,8 @@ with ExitStack() as stack: ("unrelated", "repository-other", None), ]: add_target(name, stored, live) + if name == "legacy": + state.ensure_security_target(connection, paths[name]) add_scan(name + "-scan", name, "missing" if name == "unverified" else "current") for name, identity in [("fresh", "repository-current"), ("empty", "repository-empty")]: add_target(name, identity) @@ -690,6 +693,153 @@ with ExitStack() as stack: "SELECT target_inode FROM scans WHERE id = 'newer-scan'" ).fetchone()[0] == metadata[paths["newer"]].st_ino, })) + elif scenario == "late-null": + add_target("requested", "repository-current") + add_scan("requested-scan", "requested") + add_target("unscanned", None, "repository-current") + add_target("historical", None, "repository-current") + add_scan("historical-scan", "historical") + add_finding("historical-scan", "historical-finding", closed=True) + probes.clear() + backfill = Mock(wraps=state.backfill_security_targets) + apply_migrations(connection, MIGRATIONS, lambda: timestamp, backfill) + apply_migrations(connection, MIGRATIONS, lambda: timestamp, backfill) + maintenance_probes = dict(probes) + historical_id = state.ensure_security_target(connection, paths["historical"]) + add_scan("established-scan", "unscanned") + missing.update((paths["unscanned"], paths["historical"])) + print(json.dumps({ + "stored": {row["id"]: row["repository_identity"] for row in connection.execute( + "SELECT id, repository_identity FROM security_targets" + )}, + "fullBackfillCalls": backfill.call_count, + "maintenanceProbes": {name: maintenance_probes.get(path, 0) for name, path in paths.items()}, + "historicalRegistrationId": historical_id, + "aliases": sorted(indexes.repository_target_ids(connection, "requested")), + "scans": listed("requested"), + "historicalExact": listed("historical"), + "feedback": get_scan_feedback( + connection, connection.execute("SELECT * FROM scans WHERE id = 'requested-scan'").fetchone() + )["falsePositives"], + })) + elif scenario == "binding": + for name, stored in ( + ("eligible", None), ("id-history", None), ("path-history", None), + ("donor", None), ("bound", "repository-bound"), + ): + add_target(name, stored, "repository-current") + add_scan("id-history-scan", "id-history") + connection.execute("UPDATE scans SET target_path = ? WHERE id = 'id-history-scan'", (paths["donor"],)) + add_scan("path-history-scan", "donor") + connection.execute("UPDATE scans SET target_path = ? WHERE id = 'path-history-scan'", (paths["path-history"],)) + def bind(name, path=None): + return state._bind_unscanned_repository_identity( + connection, name, path or paths[name], "repository-current" + ) + guarded = { + "idHistory": bind("id-history"), + "pathHistory": bind("path-history"), + "wrongPath": bind("eligible", paths["donor"]), + "alreadyBound": bind("bound"), + "eligible": bind("eligible"), + "repeat": bind("eligible"), + } + path = paths["eligible"] + def inspected(target_id, stored=None, *, historical=False, owner=True): + return state.RepositoryTargetState( + target_id, path, stored, resolved_path=path, repository=details[path], + ownership_matches=owner, strict_owner_matches=True, + generation_predates_history=True, has_historical_scans=historical, + ) + def cursor(row): + return SimpleNamespace(fetchone=lambda: row) + ignored = Mock() + ignored.execute.side_effect = [cursor(None), None, cursor({ + "id": "actual-target", "repository_identity": None, + })] + with patch.object(state, "supports_repository_identity", return_value=True), \ + patch.object(state, "stable_target_id", return_value="proposed-target"), \ + patch.object(state, "_inspect_repository_target", return_value=inspected("actual-target", historical=True)) as inspect, \ + patch.object(state, "_bind_unscanned_repository_identity") as guarded_bind: + ignored_id = state.ensure_security_target(ignored, path) + ignored_reselected = inspect.call_args.args[1:] == ("actual-target", path, None) + ignored_unbound = guarded_bind.call_count == 0 + insert = ignored.execute.call_args_list[1].args + def lost_binding(stored, owner): + database = Mock() + database.execute.side_effect = [ + cursor({"id": "actual-target", "repository_identity": None}), + cursor({"id": "actual-target", "repository_identity": stored}), + ] + with patch.object(state, "supports_repository_identity", return_value=True), \ + patch.object(state, "_inspect_repository_target", side_effect=[ + inspected("actual-target"), + inspected("actual-target", stored, historical=True, owner=owner), + ]) as inspect, \ + patch.object(state, "_bind_unscanned_repository_identity", return_value=False) as guarded_bind: + try: + accepted = state.ensure_security_target(database, path) == "actual-target" + except SystemExit: + accepted = False + return { + "accepted": accepted, + "rechecked": inspect.call_count == 2 and inspect.call_args.args[1:] == ("actual-target", path, stored), + "guarded": guarded_bind.call_args.args[1:] == ("actual-target", path, "repository-current"), + } + print(json.dumps({ + "guarded": guarded, + "stored": {row["id"]: row["repository_identity"] for row in connection.execute( + "SELECT id, repository_identity FROM security_targets" + )}, + "ignoredInsert": { + "id": ignored_id, "reselected": ignored_reselected, "unbound": ignored_unbound, + "insertsNull": "INSERT OR IGNORE" in insert[0] and "repository_identity" not in insert[0] and len(insert[1]) == 5, + }, + "lostToHistory": lost_binding(None, True), + "lostToConflictingIdentity": lost_binding("repository-other", False), + })) + elif scenario == "sealed-comparison": + for name, stored in (("legacy", None), ("first", "repository-saved"), ("second", "repository-saved")): + add_target(name, stored) + for scan_id, target in (("legacy-before", "legacy"), ("legacy-after", "legacy"), ("first-scan", "first"), ("second-scan", "second")): + add_scan(scan_id, target) + resolution_errors.update({paths["legacy"]: RuntimeError, paths["first"]: OSError, paths["second"]: RuntimeError}) + connection.commit() + require_scan = lambda database, scan_id: database.execute("SELECT * FROM scans WHERE id = ?", (scan_id,)).fetchone() + coverage = lambda scan: {"completeness": "complete"} + accepted = [] + with patch.object(state, "_inspect_repository_target", side_effect=AssertionError("Sealed comparison inspected the live target")): + for before, after in (("legacy-before", "legacy-after"), ("first-scan", "second-scan")): + args = argparse.Namespace(before_scan_id=before, after_scan_id=after, matches_json='{"matches":[],"uncertain":[]}') + compared = history.compare_scans(connection, args, require_scan=require_scan, read_coverage=coverage) + saved = history.save_scan_comparison(connection, args, now=lambda: timestamp, require_scan=require_scan, read_coverage=coverage) + accepted.append(compared["afterScanId"] == after and saved["beforeScanId"] == before) + conflicting = history._same_repository( + connection, + {"target_id": "same-target", "repository_identity": "repository-first"}, + {"target_id": "same-target", "repository_identity": "repository-second"}, + identities=Mock(), + ) + add_target("unregistered", None) + connection.execute("DELETE FROM security_targets WHERE id = 'unregistered'") + refused = state.RepositoryTargetState("", paths["unregistered"], None) + cache = Mock(supports_identity=True) + cache.for_path.return_value = refused + with patch.object(history, "RepositoryIdentityCache", return_value=cache): + try: + history.list_unmatched_scan_pairs( + connection, argparse.Namespace(repository=paths["unregistered"], force=False), + backfill_finding_details=lambda *_: None, read_coverage=coverage, + ) + except SystemExit: + automatic_rejected = True + else: + automatic_rejected = False + print(json.dumps({ + "accepted": accepted, "conflictingIdentitiesAccepted": conflicting, + "savedCount": connection.execute("SELECT COUNT(*) FROM scan_comparisons").fetchone()[0], + "unregisteredAutomaticRequesterRejected": automatic_rejected, + })) else: stack.enter_context(patch.object(os.path, "normcase", lambda value: os.fspath(value).lower())) originals = {} @@ -790,7 +940,7 @@ test("reads precise Linux birth time through supported native interfaces", () => } }); -test("reuses verified legacy aliases and probes each saved target once per request", () => { +test("reuses established aliases and probes each saved target once per request", () => { const result = run("cache"); expect(result["scans"]).toEqual([ @@ -827,7 +977,7 @@ test("reuses verified legacy aliases and probes each saved target once per reque expect(result["matchingUnrelatedOrigins"]).toBe(1); expect(result["indexingMaxProbes"]).toBe(1); expect(result["feedbackMaxProbes"]).toBe(1); - expect(result["legacyStored"]).toBeNull(); + expect(result["legacyStored"]).toBe("repository-current"); expect(result["changedStored"]).toBe("repository-previous"); }); @@ -909,12 +1059,12 @@ test("upgrades only independently verified pre-release repository hashes", () => ); }); -test("does not assign legacy history to a newer or indeterminate Git generation", () => { +test("keeps unproved legacy history unbound and rejects newer or indeterminate owners", () => { const result = run("null-history"); expect(result["sameCheckoutMetadata"]).toBe(true); expect(result["stored"]).toEqual({ - unchanged: "current-unchanged", + unchanged: null, newer: null, "invalid-time": null, "no-history": "current-no-history", @@ -924,13 +1074,79 @@ test("does not assign legacy history to a newer or indeterminate Git generation" expect(result["aliases"]).toEqual(["replacement-alias"]); }); +test("revisits only eligible late NULL identities on ordinary database opens", () => { + const result = run("late-null"); + + expect(result["stored"]).toEqual({ + requested: "repository-current", + unscanned: "repository-current", + historical: null, + }); + expect(result["fullBackfillCalls"]).toBe(0); + expect(result["maintenanceProbes"]).toEqual({ + requested: 0, + unscanned: 1, + historical: 0, + }); + expect(result["historicalRegistrationId"]).toBe("historical"); + expect(result["aliases"]).toEqual(["requested", "unscanned"]); + expect(result["scans"]).toEqual(["established-scan", "requested-scan"]); + expect(result["historicalExact"]).toEqual(["historical-scan"]); + expect(result["feedback"]).toEqual([]); +}); + +test("guards identity binding at the database write and rechecks lost bindings", () => { + const result = run("binding"); + + expect(result["guarded"]).toEqual({ + idHistory: false, + pathHistory: false, + wrongPath: false, + alreadyBound: false, + eligible: true, + repeat: false, + }); + expect(result["stored"]).toEqual({ + eligible: "repository-current", + "id-history": null, + "path-history": null, + donor: null, + bound: "repository-bound", + }); + expect(result["ignoredInsert"]).toEqual({ + id: "actual-target", + reselected: true, + unbound: true, + insertsNull: true, + }); + expect(result["lostToHistory"]).toEqual({ + accepted: true, + rechecked: true, + guarded: true, + }); + expect(result["lostToConflictingIdentity"]).toEqual({ + accepted: false, + rechecked: true, + guarded: true, + }); +}); + +test("compares sealed history from persisted evidence without weakening automatic requesters", () => { + const result = run("sealed-comparison"); + + expect(result["accepted"]).toEqual([true, true]); + expect(result["conflictingIdentitiesAccepted"]).toBe(false); + expect(result["savedCount"]).toBe(2); + expect(result["unregisteredAutomaticRequesterRejected"]).toBe(true); +}); + test("admits rerun lineage only through the verified repository and exact scope", () => { const result = run("lineage"); expect(result["accepted"]).toEqual({ requested: true, removed: true, - legacy: true, + legacy: false, unverified: false, clone: false, scope: false, diff --git a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts index 5c75f0611..dcffa8eaf 100644 --- a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts +++ b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts @@ -787,10 +787,10 @@ describe("durable workbench repository identities", () => { expect(result["explicitCloneCompared"]).toBe(true); }, 30_000); - test("backfills only current owners and preserves every existing target ID", () => { + test("leaves unproved historical identities unbound and preserves every target ID", () => { const result = runProbe("backfill", fixture()); - expect(result["valid"]).toMatch(/^repository_sha256_[a-f0-9]{64}$/); + expect(result["valid"]).toBeNull(); expect(result["reused"]).toBeNull(); expect(result["mixed"]).toBeNull(); expect(result["malformed"]).toBeNull(); @@ -839,12 +839,13 @@ describe("durable workbench repository identities", () => { }; expect(result["persisted"]).toEqual(verified); - expect(result["verifiedLegacy"]).toEqual(verified); - expect(result["unverifiedLegacy"]).toEqual({ + const isolated = { scans: ["requested-scan"], matchingScanCount: 1, matchingScanIds: [], - }); + }; + expect(result["verifiedLegacy"]).toEqual(isolated); + expect(result["unverifiedLegacy"]).toEqual(isolated); expect(result["candidateIdentityPreserved"]).toBe(true); }, 30_000); From e4694eb53c7e4257c0509f153e12b0eb0d3a1985 Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sun, 16 Aug 2026 07:28:32 -0700 Subject: [PATCH 14/21] Bind repository history to each scan generation --- sdk/typescript/README.md | 15 +- .../scripts/deep_scan_workbench.py | 18 +- .../_bundled_plugin/scripts/workbench_db.py | 25 +- .../scripts/workbench_feedback.py | 28 +- .../scripts/workbench_native_indexes.py | 91 +++-- .../scripts/workbench_scan_history.py | 116 ++----- .../scripts/workbench_scan_start.py | 7 +- .../scripts/workbench_schema.py | 17 +- .../scripts/workbench_target_state.py | 274 +++++++++++---- .../tests-ts/workbench-identity-cache.test.ts | 313 ++++++++++++++++-- .../workbench-repository-identity.test.ts | 25 +- 11 files changed, 677 insertions(+), 252 deletions(-) diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 3834218fd..8da806da1 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -666,12 +666,15 @@ const directPublication = await publishScan("/path/to/completed-scan", { ### Scan history and reruns `scans` or `scans list` lists scans for the current repository. Linked Git -worktrees are discovered automatically and grouped when they share the selected -state database. History written without a recorded Git generation stays attached -to its original target and scan IDs. If an older client creates and scans a -target before its identity can be recorded, later clients do not retroactively -share that history across worktrees. Findings indicate whether they were confirmed -in the repository's latest completed scan across those linked worktrees. Finding confirmation and +worktrees are discovered automatically when their scans record the same Git +generation in the selected state database. The generation identifies the shared +Git and primary object-store directory instances; it is not a repository UUID. +History written without that scan-level evidence stays attached to its original +target and scan IDs. Older clients' history is not retroactively shared across +worktrees. Explicit comparisons between independently verified copies of the same +repository remain available without joining their finding histories. Findings +indicate whether they were confirmed in the repository's latest completed scan +across those linked worktrees. Finding confirmation and automatic matching use the order in which completed scans become visible in the workbench; sealed report timestamps are unchanged. Pass a repository path to inspect another checkout. `--scan-root DIR` only filters scans already recorded diff --git a/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py b/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py index 113f7c156..7085bcc7a 100644 --- a/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py +++ b/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py @@ -25,6 +25,7 @@ git_revision, worktree_content_digest, ) +from workbench_target_state import RegisteredRepositoryTarget from workbench_validation import optional_text, require_uuid, user_text DEEP_SCAN_WORKER_KINDS = ("setup", "discovery", "dedup") @@ -135,7 +136,7 @@ class DeepScanDependencies: require_remediation_target: Callable[[str], Path] require_scannable_target: Callable[[Path], None] require_scope: Callable[[str, str, Path], str] - ensure_security_target: Callable[[sqlite3.Connection, str], str] + register_security_target: Callable[[sqlite3.Connection, str], RegisteredRepositoryTarget] require_canonical_scan_directory: Callable[[Path], Path] safe_segment: Callable[[str], str] compact_timestamp: Callable[[], str] @@ -188,8 +189,10 @@ def require_scope(scope: str, mode: str, target: Path) -> str: return dependencies().require_scope(scope, mode, target) -def ensure_security_target(connection: sqlite3.Connection, target_path: str) -> str: - return dependencies().ensure_security_target(connection, target_path) +def register_security_target( + connection: sqlite3.Connection, target_path: str +) -> RegisteredRepositoryTarget: + return dependencies().register_security_target(connection, target_path) def require_canonical_scan_directory(scan_dir: Path) -> Path: @@ -769,7 +772,8 @@ def begin_deep_scan_for_target( workspace_id = str(uuid.uuid4()) scan_id = str(uuid.uuid4()) timestamp = now() - target_id = ensure_security_target(connection, target_path) + registration = register_security_target(connection, target_path) + target_id = registration.target_id scan_dir = Path( tempfile.mkdtemp( prefix=f"{safe_segment(revision)}_{compact_timestamp()}_", @@ -798,17 +802,19 @@ def begin_deep_scan_for_target( connection.execute( """ INSERT INTO scans ( - id, workspace_id, target_id, target_path, target_revision, target_snapshot_digest, + id, workspace_id, target_id, repository_generation, target_path, + target_revision, target_snapshot_digest, target_device, target_inode, scope, mode, user_context, deep_scan_owner_thread_id, scan_dir, model, reasoning_effort, status, phase, handoff_status, started_at, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'deep', ?, ?, ?, ?, ?, + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'deep', ?, ?, ?, ?, ?, 'running', 'preflight', 'delivered', ?, ?, ?) """, ( scan_id, workspace_id, target_id, + registration.repository_generation, target_path, revision, target_snapshot_digest, diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index e094baa80..49733c11e 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -117,7 +117,12 @@ worktree_content_digest, worktree_content_digest_for_context, ) -from workbench_target_state import backfill_security_targets, ensure_security_target +from workbench_target_state import ( + RepositoryScanScope, + backfill_security_targets, + ensure_security_target, + register_security_target, +) from workbench_validation import ( bounded_output_text, optional_text, @@ -885,7 +890,8 @@ def start_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict "This Codex thread already has an active Deep Scan for the selected " "target and scope. Rejoin that scan instead of starting another one." ) - if ensure_security_target(connection, str(current_target)) != workspace["target_id"]: + registration = register_security_target(connection, str(current_target)) + if registration.target_id != workspace["target_id"]: raise SystemExit( "The saved workspace no longer matches the selected repository target." ) @@ -897,6 +903,7 @@ def start_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict scope=scope, diff_target=diff_target, target_identity=target_identity, + repository_generation=registration.repository_generation, target_root=target_root, target_summary=target_summary, scope_file_count=scope_file_count, @@ -1019,7 +1026,8 @@ def _start_prompt_driven_scan( workspace_id = str(uuid.uuid4()) scan_id = str(uuid.uuid4()) timestamp = now() - target_id = ensure_security_target(connection, target_path) + registration = register_security_target(connection, target_path) + target_id = registration.target_id connection.execute( """ INSERT INTO workspaces ( @@ -1052,6 +1060,7 @@ def _start_prompt_driven_scan( scope=scope, diff_target=diff_target, target_identity=target_identity, + repository_generation=registration.repository_generation, target_root=target_root, target_summary=target_summary, scope_file_count=scope_file_count, @@ -1652,10 +1661,13 @@ def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace) connection.execute("BEGIN IMMEDIATE") try: archive_scan(connection, args, scan_dir, timestamp, require_canonical_scan_directory) - target_id = ensure_security_target(connection, str(repository)) + registration = register_security_target(connection, str(repository)) + target_id = registration.target_id if parent_scan_id is not None: parent = require_scan(connection, parent_scan_id) - if parent["target_id"] not in native_indexes.repository_target_ids(connection, target_id): + if not RepositoryScanScope( + registration.repository_generation, target_id + ).contains(parent): raise SystemExit("A rerun must belong to the same repository as its parent scan.") connection.execute( @@ -1687,6 +1699,7 @@ def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace) scope=scope, diff_target=diff_target, target_identity=target_identity, + repository_generation=registration.repository_generation, target_root=scan_dir.parent, target_summary=None, scope_file_count=scope_file_count, @@ -3833,7 +3846,7 @@ def main() -> None: require_remediation_target=require_remediation_target, require_scannable_target=require_scannable_target, require_scope=require_scope, - ensure_security_target=ensure_security_target, + register_security_target=register_security_target, require_canonical_scan_directory=require_canonical_scan_directory, safe_segment=safe_segment, compact_timestamp=compact_timestamp, diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py b/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py index 0b1b525a1..283336836 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_feedback.py @@ -17,24 +17,29 @@ FINDING_TITLE_BYTES, ) from workbench_native_indexes import _indexed_findings -from workbench_target_state import RepositoryIdentityCache +from workbench_target_state import RepositoryIdentityCache, scan_repository_group from workbench_validation import bounded_output_text def get_scan_feedback(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict[str, Any]: identities = RepositoryIdentityCache(connection) - target_ids = sorted(identities.target_ids(scan["target_id"])) - if not target_ids: + scope = identities.scope_for_scan(scan) + if not scope.available: return {"scanId": scan["id"], "targetId": scan["target_id"], "falsePositives": []} indexed_findings = { - finding_id: finding + (scan_repository_group(finding), finding_id): finding for finding in _indexed_findings( - connection, identities=identities, target_ids=set(target_ids) + connection, identities=identities, scan_scope=scope ) for finding_id in finding["matched_finding_ids"] } - target_placeholders = ", ".join("?" for _ in target_ids) + source_filter, source_values = scope.sql( + "source_scans", supports_generation=identities.supports_generation + ) + source_generation = ( + "source_scans.repository_generation" if identities.supports_generation else "NULL" + ) rows = connection.execute( f""" WITH ranked_decisions AS ( @@ -44,10 +49,13 @@ def get_scan_feedback(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict triage.close_reason, triage.note, COALESCE(triage.updated_at, source_scans.completed_at) AS updated_at, source_scans.id AS source_scan_id, + source_scans.target_id, + {source_generation} AS repository_generation, source_scans.completed_at AS source_completed_at, locations.relative_path, locations.start_line, locations.end_line, locations.role, ROW_NUMBER() OVER ( - PARTITION BY findings.id + PARTITION BY findings.id, {source_generation}, + CASE WHEN {source_generation} IS NULL THEN source_scans.target_id END ORDER BY COALESCE(triage.updated_at, source_scans.completed_at) DESC, source_scans.completed_at DESC, source_scans.id DESC, occurrences.id DESC @@ -64,7 +72,7 @@ def get_scan_feedback(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict candidate.sort_order LIMIT 1 ) - WHERE source_scans.target_id IN ({target_placeholders}) + WHERE {source_filter} AND source_scans.id != ? AND source_scans.status = 'complete' ) @@ -77,12 +85,12 @@ def get_scan_feedback(connection: sqlite3.Connection, scan: sqlite3.Row) -> dict AND trim(note) != '' ORDER BY updated_at DESC, source_completed_at DESC, source_scan_id DESC, finding_id DESC """, - (*target_ids, scan["id"]), + (*source_values, scan["id"]), ) false_positives = [] reviewed_components: set[str] = set() for row in rows: - finding = indexed_findings.get(row["finding_id"]) + finding = indexed_findings.get((scan_repository_group(row), row["finding_id"])) if ( finding is None or finding["status"] != "closed" diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py index a56166dca..cf67b5c5e 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py @@ -3,7 +3,6 @@ import argparse import sqlite3 import sys -from collections import Counter from collections.abc import Iterator from itertools import islice from pathlib import Path @@ -13,7 +12,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import workbench_scan_history as scan_history from workbench_constants import FINDING_SUMMARY_BYTES, FINDING_TITLE_BYTES, FINDINGS_PAGE_MAX -from workbench_target_state import RepositoryIdentityCache +from workbench_target_state import ( + RepositoryIdentityCache, + RepositoryScanScope, + scan_repository_group, +) from workbench_validation import bounded_output_text @@ -38,17 +41,17 @@ def list_global_findings( if repository is not None: repository = str(Path(repository).expanduser().resolve()) requested = identities.for_path(repository) - target_ids = identities.target_ids_for_path(repository) + scan_scope = identities.scope_for_path(repository) else: - target_ids = ( - None if args.target_id is None else identities.target_ids(args.target_id) + scan_scope = ( + None if args.target_id is None else identities.scope(args.target_id) ) target = identities.targets.get(args.target_id) if identities.supports_identity and target is not None: requested = identities.for_row(target) findings = ( row - for row in _indexed_findings(connection, identities=identities, target_ids=target_ids) + for row in _indexed_findings(connection, identities=identities, scan_scope=scan_scope) if (args.severity is None or row["severity"] == args.severity) and (args.status is None or row["status"] == args.status) and ( @@ -102,19 +105,21 @@ def _indexed_findings( connection: sqlite3.Connection, *, identities: RepositoryIdentityCache | None = None, - target_ids: set[str] | None = None, + scan_scope: RepositoryScanScope | None = None, ) -> Iterator[dict[str, Any]]: - if target_ids is not None and not target_ids: - return identities = identities or RepositoryIdentityCache(connection) - target_values = tuple(sorted(target_ids)) if target_ids is not None else () - placeholders = ", ".join("?" for _ in target_values) - target_filter = f"scans.target_id IN ({placeholders})" if target_ids is not None else "1" - match_filter = ( - f"before_scans.target_id IN ({placeholders}) " - f"AND after_scans.target_id IN ({placeholders})" - if target_ids is not None else "1" - ) + + def scope_sql(alias: str) -> tuple[str, tuple[str, ...]]: + return scan_scope.sql( + alias, supports_generation=identities.supports_generation + ) if scan_scope is not None else ("1", ()) + + def generation_sql(alias: str) -> str: + return f"{alias}.repository_generation" if identities.supports_generation else "NULL" + + target_filter, target_values = scope_sql("scans") + before_filter, before_values = scope_sql("before_scans") + after_filter, after_values = scope_sql("after_scans") parents: dict[ tuple[tuple[str, str], str], tuple[tuple[str, str], str] ] = {} @@ -128,6 +133,8 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: f""" SELECT before_scans.target_id AS before_target_id, after_scans.target_id AS after_target_id, + {generation_sql("before_scans")} AS before_generation, + {generation_sql("after_scans")} AS after_generation, before.finding_id AS before_finding_id, after.finding_id AS after_finding_id FROM scan_comparison_matches AS matches @@ -137,12 +144,18 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: JOIN finding_occurrences AS after ON after.id = matches.after_occurrence_id JOIN scans AS after_scans ON after_scans.id = after.scan_id JOIN security_targets AS after_targets ON after_targets.id = after_scans.target_id - WHERE {match_filter} + WHERE {before_filter} AND {after_filter} """, - (*target_values, *target_values), + (*before_values, *after_values), ): - before_repository = identities.group(match["before_target_id"]) - after_repository = identities.group(match["after_target_id"]) + before_repository = scan_repository_group({ + "target_id": match["before_target_id"], + "repository_generation": match["before_generation"], + }) + after_repository = scan_repository_group({ + "target_id": match["after_target_id"], + "repository_generation": match["after_generation"], + }) if before_repository != after_repository: continue before = group( @@ -168,11 +181,12 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: "scans.completed_at" if "completed_at" in identities.scan_columns else "NULL" ) latest_scan_by_repository = { - identities.group(row["target_id"]): row["id"] + scan_repository_group(row): row["id"] for row in sorted( connection.execute( f""" SELECT scans.target_id, scans.id, scans.started_at, + {generation_sql("scans")} AS repository_generation, {completed_at_column} AS completed_at, {completion_column} AS completion_sequence FROM scans @@ -196,6 +210,7 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: scans.id AS scan_id, scans.started_at AS scan_started_at, scans.target_id, + {generation_sql("scans")} AS repository_generation, targets.current_path AS target_path, scans.scope, MAX(scans.updated_at, COALESCE(triage.updated_at, '')) AS updated_at, @@ -224,7 +239,7 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: grouped.setdefault( group( ( - identities.group(row["target_id"]), + scan_repository_group(row), row["finding_id"], ) ), @@ -247,7 +262,7 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: ): status = "open" scans = sorted({(row["scan_started_at"], row["scan_id"]) for row in occurrences}) - latest_scan_id = latest_scan_by_repository.get(identities.group(latest["target_id"])) + latest_scan_id = latest_scan_by_repository.get(scan_repository_group(latest)) findings.append( { **dict(latest), @@ -299,19 +314,33 @@ def list_repositories( targets = {row["id"]: row for row in connection.execute("SELECT * FROM security_targets")} - open_findings_by_repository = Counter( - identities.group(row["target_id"]) - for row in _indexed_findings(connection, identities=identities) + scan_evidence = { + row["id"]: row for row in connection.execute("SELECT * FROM scans") + } + open_findings = [ + row for row in _indexed_findings(connection, identities=identities) if row["status"] == "open" - ) + ] + + def open_findings_count(target_id: str) -> int: + scope = identities.scope(target_id) + if scope.exact_target: + return sum( + row["status"] == "open" + for row in _indexed_findings( + connection, identities=identities, scan_scope=scope + ) + ) + return sum( + any(scope.contains(scan_evidence[scan_id]) for scan_id in row["known_scan_ids"]) + for row in open_findings + ) repositories = [ { "checkoutAvailable": Path(target["current_path"]).is_dir(), "displayName": target["display_name"], "latestScan": latest_scan, - "openFindingsCount": open_findings_by_repository.get( - identities.group(target_id), 0 - ), + "openFindingsCount": open_findings_count(target_id), "scanCount": scan_count_by_target[target_id], "targetId": target_id, "targetPath": target["current_path"], diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py index 613d4e2c1..d6d2cb67a 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py @@ -11,13 +11,14 @@ # Some plugin hosts launch Python with safe-path isolation enabled. sys.path.insert(0, str(Path(__file__).resolve().parent)) -from filesystem_identity import serialize_filesystem_identity from report_projection import SEVERITY_ORDER from workbench_constants import FINDINGS_PAGE_MAX from workbench_scan_usage import stored_scan_cost_fields from workbench_target_state import ( RepositoryIdentityCache, _timestamp_ns, + scan_repository_generation, + scan_repository_group, supports_repository_identity, ) @@ -29,12 +30,8 @@ def _same_repository( *, identities: RepositoryIdentityCache | None = None, ) -> bool: - before_stored = ( - before["repository_identity"] if "repository_identity" in before.keys() else None - ) - after_stored = ( - after["repository_identity"] if "repository_identity" in after.keys() else None - ) + before_stored = scan_repository_generation(before) + after_stored = scan_repository_generation(after) if before_stored is not None and before_stored == after_stored: return True if before["target_id"] and before["target_id"] == after["target_id"]: @@ -42,16 +39,21 @@ def _same_repository( identities = identities or RepositoryIdentityCache(connection) before_state = identities.for_row(before) after_state = identities.for_row(after) - if before_state.resolved_path is None or after_state.resolved_path is None: + if ( + not before_state.ownership_matches or not after_state.ownership_matches + or before_stored is not None and before_stored != before_state.live_identity + or after_stored is not None and after_stored != after_state.live_identity + ): return False - before_group = before_stored or before_state.verified_identity - after_group = after_stored or after_state.verified_identity - if before_group is not None and before_group == after_group: - return True - if before_state.resolved_path == after_state.resolved_path: + if ( + before_state.resolved_path is not None + and before_state.resolved_path == after_state.resolved_path + ): return before_stored is None or after_stored is None - before_identity = before_state.verified_identity - after_identity = after_state.verified_identity + if before_state.repository is None or after_state.repository is None: + return False + before_identity = before_state.live_identity + after_identity = after_state.live_identity if before_identity is not None and before_identity == after_identity: return True if before_identity is None or after_identity is None: @@ -72,33 +74,11 @@ def list_scans( if args is not None and args.repository: repository = Path(args.repository).expanduser().resolve() identities = RepositoryIdentityCache(connection) - requested = identities.for_path(str(repository)) - check_ownership = {"target_device", "target_inode"} <= identities.scan_columns - identity_matches = ( - requested.resolved_path is not None - and ( - requested.stored_identity is None - or requested.missing - or requested.stored_identity == requested.live_identity - ) + clause, scope_values = identities.scope_for_path(str(repository)).sql( + supports_generation=identities.supports_generation ) - repository_clauses = [ - _owned_scan_clause( - repository, - check_ownership and not requested.relaxed_directory_ownership, - values, - ) - ] if identity_matches else ["0"] - requested_identity = requested.verified_identity - if requested_identity is not None: - for target_id in identities.targets: - if target_id == requested.target_id or identities.group(target_id) != ( - "repository", requested_identity - ): - continue - repository_clauses.append("scans.target_id = ?") - values.append(target_id) - clauses.append(f"({' OR '.join(repository_clauses)})") + clauses.append(clause) + values.extend(scope_values) if args is not None and args.scan_root: scan_root = str(Path(args.scan_root).expanduser().resolve()) prefix = scan_root.rstrip(os.sep) + os.sep @@ -211,33 +191,6 @@ def list_scans( return result -def _owned_scan_clause( - target: Path, - check_ownership: bool, - values: list[Any], -) -> str: - values.append(str(target)) - target_clause = "scans.target_path = ?" - if not check_ownership: - return target_clause - try: - metadata = target.stat() - except OSError: - return target_clause - values.extend( - ( - serialize_filesystem_identity(metadata.st_dev), - serialize_filesystem_identity(metadata.st_ino), - ) - ) - return ( - f"({target_clause} AND (" - "(scans.target_device IS NULL AND scans.target_inode IS NULL) " - "OR (scans.target_device = ? AND scans.target_inode = ?)" - "))" - ) - - def _scan_completion_order(scan: sqlite3.Row) -> tuple[int, int, str]: sequence = scan["completion_sequence"] if "completion_sequence" in scan.keys() else None if sequence is not None: @@ -261,23 +214,13 @@ def list_unmatched_scan_pairs( requested = identities.for_path(str(repository)) if identities.supports_identity: requested.require_owner() - requested_row = { - "target_id": requested.target_id, - "target_path": requested.target_path, - "repository_identity": requested.stored_identity, - } - scan_query = ( - "SELECT scans.*, targets.repository_identity " - "FROM scans LEFT JOIN security_targets AS targets ON targets.id = scans.target_id " - "WHERE scans.status = 'complete' ORDER BY scans.started_at, scans.id" - if identities.supports_identity - else "SELECT * FROM scans WHERE status = 'complete' ORDER BY started_at, id" + clause, values = identities.scope_for_path(str(repository)).sql( + supports_generation=identities.supports_generation ) - selected = [ - scan - for scan in connection.execute(scan_query) - if _same_repository(connection, scan, requested_row, identities=identities) - ] + selected = connection.execute( + "SELECT * FROM scans WHERE status = 'complete' " + f"AND {clause} ORDER BY started_at, id", values + ).fetchall() available = [] for scan in selected: @@ -302,7 +245,10 @@ def list_unmatched_scan_pairs( for index, after in enumerate(available): if focus_scan_id is not None and after["id"] != focus_scan_id: continue - candidates = available[:index] + candidates = [ + before for before in available[:index] + if scan_repository_group(before) == scan_repository_group(after) + ] previous = [ before for before in candidates diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py index 7cb1f6217..deffa8789 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py @@ -157,6 +157,7 @@ def insert_running_scan( scope: str, diff_target: dict[str, str] | None, target_identity: tuple[str, str | None, int | str, int | str], + repository_generation: str | None, target_root: Path, target_summary: str | None, scope_file_count: int, @@ -179,18 +180,20 @@ def insert_running_scan( connection.execute( """ INSERT INTO scans ( - id, workspace_id, target_id, target_path, target_revision, target_snapshot_digest, + id, workspace_id, target_id, repository_generation, target_path, + target_revision, target_snapshot_digest, target_device, target_inode, scope, mode, user_context, deep_scan_owner_thread_id, diff_target_kind, diff_base_revision, diff_head_revision, diff_content_digest, target_summary, scan_dir, model, reasoning_effort, status, phase, handoff_status, started_at, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'running', 'preflight', ?, ?, ?, ?) """, ( scan_id, workspace["id"], workspace["target_id"], + repository_generation, str(target), *target_identity, scope, diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py index ebadf69f4..719daebe4 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py @@ -693,6 +693,12 @@ CREATE INDEX security_targets_by_repository_identity ON security_targets(repository_identity); + ALTER TABLE scans + ADD COLUMN repository_generation TEXT; + + CREATE INDEX scans_by_repository_generation + ON scans(repository_generation); + ALTER TABLE scans ADD COLUMN completion_sequence INTEGER CHECK (completion_sequence >= 1); @@ -1339,10 +1345,13 @@ def repair_repository_identity_migration(connection: sqlite3.Connection) -> bool add_column_if_missing(connection, "security_targets", "repository_identity", "TEXT") continue if statement.startswith("ALTER TABLE scans"): - add_column_if_missing( - connection, "scans", "completion_sequence", - "INTEGER CHECK (completion_sequence >= 1)", - ) + if "ADD COLUMN repository_generation " in statement: + add_column_if_missing(connection, "scans", "repository_generation", "TEXT") + else: + add_column_if_missing( + connection, "scans", "completion_sequence", + "INTEGER CHECK (completion_sequence >= 1)", + ) continue for prefix in ("CREATE UNIQUE INDEX ", "CREATE INDEX ", "CREATE TRIGGER "): if statement.startswith(prefix): diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index 76784b68d..cd3f11f15 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -22,7 +22,7 @@ serialize_filesystem_identity, stored_filesystem_identity_matches, ) -from workbench_target import git_output +from workbench_target import git_bytes, git_output def stable_target_id(target: Path) -> str: @@ -37,7 +37,7 @@ def repository_relative_path(target: Path) -> str | None: def _repository_worktree(target: Path) -> tuple[Path, str] | None: - worktree_root = git_output(target, "rev-parse", "--show-toplevel") + worktree_root = _git_path(target, "rev-parse", "--show-toplevel") if worktree_root is None: return None try: @@ -48,6 +48,53 @@ def _repository_worktree(target: Path) -> tuple[Path, str] | None: return canonical_root, relative.as_posix() +def _path_from_git_bytes(value: bytes, relative_to: Path) -> Path | None: + if value.endswith(b"\n"): + value = value[:-1] + if not value or b"\0" in value: + return None + path = Path(os.fsdecode(value)) + return Path(os.path.realpath(path if path.is_absolute() else relative_to / path)) + + +def _git_path(target: Path, *args: str) -> Path | None: + value = git_bytes(target, *args) + return _path_from_git_bytes(value, target) if value is not None else None + + +def _registered_worktree(target: Path, root: Path, common: Path) -> bool: + registered = git_bytes(target, "worktree", "list", "--porcelain", "-z") + if registered is not None: + return any( + os.path.realpath(os.fsdecode(record[len(b"worktree "):])) == str(root) + for record in registered.split(b"\0") + if record.startswith(b"worktree ") + ) + # Git before 2.36 emits unquoted newline-delimited paths. Its reciprocal + # administrative files are unambiguous even when a checkout path has newlines. + gitdir = _git_path(target, "rev-parse", "--absolute-git-dir") + dotgit = root / ".git" + try: + mode = dotgit.lstat().st_mode + if stat.S_ISDIR(mode): + return gitdir == common == Path(os.path.realpath(dotgit)) + if ( + not stat.S_ISREG(mode) or gitdir is None + or gitdir.parent != common / "worktrees" or not gitdir.is_dir() + ): + return False + forward = dotgit.read_bytes() + if not forward.startswith(b"gitdir: "): + return False + return ( + _path_from_git_bytes(forward[len(b"gitdir: "):], root) == gitdir + and _path_from_git_bytes((gitdir / "gitdir").read_bytes(), gitdir) + == Path(os.path.realpath(dotgit)) + ) + except (OSError, ValueError): + return False + + class _LinuxStatxTimestamp(ctypes.Structure): _fields_ = ( ("seconds", ctypes.c_int64), @@ -155,14 +202,21 @@ class GitRepositoryIdentity: inode: int | str birth_time_ns: int + @property + def legacy_value(self) -> str: + return _identity_digest( + f"git-common-dir\0{self.common_directory}\0{self.device}\0{self.inode}\0" + f"{self.birth_time_ns}\0{self.relative_path}" + ) + def _identity_digest(material: str) -> str: - return f"repository_sha256_{hashlib.sha256(material.encode()).hexdigest()}" + return f"repository_sha256_{hashlib.sha256(material.encode(errors='surrogateescape')).hexdigest()}" def _repository_identity_details(target: Path | str) -> GitRepositoryIdentity | None: target = Path(target) - common_directory = git_output( + common_directory = _git_path( target, "rev-parse", "--path-format=absolute", "--git-common-dir" ) if common_directory is None: @@ -171,29 +225,33 @@ def _repository_identity_details(target: Path | str) -> GitRepositoryIdentity | if worktree is None: return None worktree_root, relative = worktree - registered = git_output(target, "worktree", "list", "--porcelain", "-z") - if registered is None: + if not _registered_worktree(target, worktree_root, common_directory): return None - canonical_root = os.fspath(worktree_root) - if not any( - os.path.realpath(record.removeprefix("worktree ")) == canonical_root - for record in registered.split("\0") - if record.startswith("worktree ") - ): + object_directory = _git_path( + target, "rev-parse", "--path-format=absolute", "--git-path", "objects" + ) + if object_directory is None: return None - canonical_directory = os.path.realpath(common_directory) + canonical_directory = str(common_directory) try: - metadata = Path(canonical_directory).stat() + metadata = common_directory.stat() + object_metadata = object_directory.stat() except OSError: return None + if not stat.S_ISDIR(metadata.st_mode) or not stat.S_ISDIR(object_metadata.st_mode): + return None birth_time_ns = _repository_birth_time_ns(canonical_directory, metadata) - if birth_time_ns is None: + object_birth_time_ns = _repository_birth_time_ns(str(object_directory), object_metadata) + if birth_time_ns is None or object_birth_time_ns is None: return None device = serialize_filesystem_identity(metadata.st_dev) inode = serialize_filesystem_identity(metadata.st_ino) material = ( - f"git-common-dir\0{canonical_directory}\0{device}\0{inode}\0" - f"{birth_time_ns}\0{relative}" + f"git-generation-v2\0{canonical_directory}\0{device}\0{inode}\0" + f"{birth_time_ns}\0{object_directory}\0" + f"{serialize_filesystem_identity(object_metadata.st_dev)}\0" + f"{serialize_filesystem_identity(object_metadata.st_ino)}\0" + f"{object_birth_time_ns}\0{relative}" ) return GitRepositoryIdentity( _identity_digest(material), relative, canonical_directory, device, inode, birth_time_ns @@ -273,15 +331,6 @@ def verified_identity(self) -> str | None: ) return self.live_identity if self.live_identity == self.stored_identity else None - @property - def relaxed_directory_ownership(self) -> bool: - return ( - self.stored_identity is not None - and self.verified_identity == self.stored_identity - and self.repository is not None - and self.repository.relative_path != "." - ) - def require_owner(self) -> None: if not self.ownership_matches: raise SystemExit( @@ -327,6 +376,16 @@ def _inspect_repository_target( generation_predates_history = False ownership_matches = True if {"target_id", "target_path"} <= scan_columns: + recorded_generations = { + row[0] for row in connection.execute( + "SELECT DISTINCT repository_generation FROM scans " + "WHERE (target_id = ? OR target_path = ?) AND repository_generation IS NOT NULL", + (target_id, target_path), + ) + } if "repository_generation" in scan_columns else set() + generation_conflict = bool(recorded_generations) and ( + repository is None or recorded_generations != {repository.value} + ) if not {"target_device", "target_inode"} <= scan_columns: historical_scan = connection.execute( "SELECT 1 FROM scans WHERE target_id = ? OR target_path = ? LIMIT 1", @@ -365,10 +424,13 @@ def _inspect_repository_target( ) or not stored_filesystem_identity_matches(inode, metadata.st_ino): mismatch = True strict_owner_matches = False + stored_matches = ( + repository is not None + and stored_identity in {repository.value, repository.legacy_value} + ) verified_repository = ( - stored_identity is not None - and repository is not None - and repository.value == stored_identity + repository is not None + and (stored_matches or recorded_generations == {repository.value}) ) ownership_matches = not ( malformed_owner @@ -380,13 +442,14 @@ def _inspect_repository_target( ) or stored_identity is not None and ( - not verified_repository or historical_scan and not recorded_owner + not stored_matches or historical_scan and not recorded_owner ) or stored_identity is None and repository is not None and historical_scan and not generation_predates_history ) + ownership_matches = ownership_matches and not generation_conflict return RepositoryTargetState( target_id, target_path, stored_identity, resolved_path, metadata, repository, ownership_matches, strict_owner_matches, generation_predates_history, historical_scan, @@ -405,6 +468,54 @@ def verified_repository_identity( ).verified_identity +def scan_repository_generation(scan: sqlite3.Row | dict) -> str | None: + return scan["repository_generation"] if "repository_generation" in scan.keys() else None + + +def scan_repository_group(scan: sqlite3.Row | dict) -> tuple[str, str]: + generation = scan_repository_generation(scan) + return ( + ("repository", generation) if generation is not None else ("target", scan["target_id"]) + ) + + +@dataclass(frozen=True) +class RepositoryScanScope: + """A verified generation and, separately, its exact target's legacy scans.""" + + generation: str | None = None + target_id: str | None = None + exact_target: bool = False + available: bool = True + + def sql( + self, alias: str = "scans", *, supports_generation: bool = True + ) -> tuple[str, tuple[str, ...]]: + if not self.available: + return "0", () + if self.exact_target: + return (f"{alias}.target_id = ?", (self.target_id,)) if self.target_id else ("0", ()) + clauses, values = [], [] + if self.generation is not None and supports_generation: + clauses.append(f"{alias}.repository_generation = ?") + values.append(self.generation) + if self.target_id: + legacy = f"{alias}.repository_generation IS NULL AND " if supports_generation else "" + clauses.append(f"({legacy}{alias}.target_id = ?)") + values.append(self.target_id) + return (f"({' OR '.join(clauses)})", tuple(values)) if clauses else ("0", ()) + + def contains(self, scan: sqlite3.Row | dict) -> bool: + if not self.available: + return False + if self.exact_target: + return bool(self.target_id) and scan["target_id"] == self.target_id + generation = scan_repository_generation(scan) + if generation is not None: + return generation == self.generation + return bool(self.target_id) and scan["target_id"] == self.target_id + + class RepositoryIdentityCache: """One request's saved identities and verified live aliases.""" @@ -414,6 +525,7 @@ def __init__(self, connection: sqlite3.Connection) -> None: self.scan_columns = { row["name"] for row in connection.execute("PRAGMA table_info(scans)") } + self.supports_generation = "repository_generation" in self.scan_columns identity_column = "repository_identity" if self.supports_identity else "NULL" self.targets = { row["target_id"]: row @@ -429,10 +541,15 @@ def __init__(self, connection: sqlite3.Connection) -> None: self._origins: dict[str, tuple[str, str] | None] = {} def for_row(self, row: sqlite3.Row | dict) -> RepositoryTargetState: + target = self.targets.get(row["target_id"]) + stored = ( + row["repository_identity"] if "repository_identity" in row.keys() + else target["repository_identity"] if target is not None else None + ) key = ( row["target_id"] or "", row["target_path"], - row["repository_identity"] if "repository_identity" in row.keys() else None, + stored, ) if key not in self._states: self._states[key] = _inspect_repository_target( @@ -448,33 +565,50 @@ def for_path(self, target_path: str) -> RepositoryTargetState: } ) - def group(self, target_id: str) -> tuple[str, str]: + def scope(self, target_id: str) -> RepositoryScanScope: target = self.targets.get(target_id) - identity = target["repository_identity"] if target is not None else None - if identity is None and target is not None: - identity = self.for_row(target).verified_identity - return ("target", target_id) if identity is None else ("repository", identity) + return ( + self._scope_for_state(self.for_row(target)) if target is not None + else RepositoryScanScope(target_id=target_id) + ) - def target_ids(self, target_id: str) -> set[str]: - target = self.targets.get(target_id) - if not self.supports_identity or target is None: - return {target_id} - return self._target_ids_for_state(self.for_row(target)) + def scope_for_path(self, target_path: str) -> RepositoryScanScope: + return self._scope_for_state(self.for_path(target_path)) - def target_ids_for_path(self, target_path: str) -> set[str]: - return self._target_ids_for_state(self.for_path(target_path)) + def scope_for_scan(self, scan: sqlite3.Row | dict) -> RepositoryScanScope: + requested = self.for_row(scan) + generation = scan_repository_generation(scan) + if not requested.ownership_matches or ( + generation is not None and generation != requested.live_identity + ): + return RepositoryScanScope(available=False) + return RepositoryScanScope(generation, requested.target_id) - def _target_ids_for_state(self, requested: RepositoryTargetState) -> set[str]: + def _scope_for_state(self, requested: RepositoryTargetState) -> RepositoryScanScope: if not requested.ownership_matches: - return set() - identity = ( - requested.stored_identity or requested.verified_identity - if self.supports_identity else None + return RepositoryScanScope(available=False) + return RepositoryScanScope( + requested.live_identity if not requested.missing else None, + requested.target_id or None, + exact_target=requested.missing, ) - if identity is None: - return {requested.target_id} if requested.target_id else set() - group = ("repository", identity) - return {candidate for candidate in self.targets if self.group(candidate) == group} + + def target_ids(self, target_id: str) -> set[str]: + return self._target_ids_for_scope(self.scope(target_id)) + + def target_ids_for_path(self, target_path: str) -> set[str]: + return self._target_ids_for_scope(self.scope_for_path(target_path)) + + def _target_ids_for_scope(self, scope: RepositoryScanScope) -> set[str]: + clause, values = scope.sql(supports_generation=self.supports_generation) + result = { + row[0] for row in self.connection.execute( + f"SELECT DISTINCT target_id FROM scans WHERE {clause}", values + ) if row[0] is not None + } + if scope.available and scope.target_id: + result.add(scope.target_id) + return result def origin(self, state: RepositoryTargetState) -> tuple[str, str] | None: if state.target_path not in self._origins: @@ -546,7 +680,8 @@ def normalize_pre_release_repository_identities(connection: sqlite3.Connection) state.stored_identity: state.repository for state in states.values() if state.repository is not None - and state.verified_identity == state.stored_identity + and state.ownership_matches + and state.stored_identity in {state.repository.value, state.repository.legacy_value} } for target in targets: stored = target["repository_identity"] @@ -571,7 +706,7 @@ def normalize_pre_release_repository_identities(connection: sqlite3.Connection) identity, scans, empty_timestamp=target["created_at"] ) ): - replacement = identity.value + replacement = identity.legacy_value if replacement != stored: connection.execute( "UPDATE security_targets SET repository_identity = ? " @@ -644,9 +779,15 @@ def backfill_security_targets(connection: sqlite3.Connection) -> None: backfill_repository_identities(connection) -def ensure_security_target( +@dataclass(frozen=True) +class RegisteredRepositoryTarget: + target_id: str + repository_generation: str | None + + +def register_security_target( connection: sqlite3.Connection, target_path: str, *, verify_ownership: bool = True -) -> str: +) -> RegisteredRepositoryTarget: supports_identity = supports_repository_identity(connection) target_query = ( "SELECT id, repository_identity FROM security_targets WHERE current_path = ?" @@ -671,7 +812,7 @@ def ensure_security_target( raise SystemExit("The repository target changed while it was being registered.") target_id = str(existing["id"]) if not supports_identity: - return target_id + return RegisteredRepositoryTarget(target_id, None) state = _inspect_repository_target( connection, target_id, target_path, existing["repository_identity"] ) @@ -687,11 +828,22 @@ def ensure_security_target( if existing is None: raise SystemExit("The repository target changed while it was being registered.") target_id = str(existing["id"]) + state = _inspect_repository_target( + connection, target_id, target_path, existing["repository_identity"] + ) if verify_ownership: - _inspect_repository_target( - connection, target_id, target_path, existing["repository_identity"] - ).require_owner() - return target_id + state.require_owner() + return RegisteredRepositoryTarget( + target_id, state.live_identity if state.ownership_matches and not state.missing else None + ) + + +def ensure_security_target( + connection: sqlite3.Connection, target_path: str, *, verify_ownership: bool = True +) -> str: + return register_security_target( + connection, target_path, verify_ownership=verify_ownership + ).target_id def main() -> None: diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts index d079d54c7..ddd5c49c4 100644 --- a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -6,6 +6,7 @@ import { PLUGIN_ROOT } from "./plugin-root.js"; const probe = String.raw` import argparse import ctypes +import errno import hashlib import io import json @@ -25,6 +26,8 @@ import workbench_native_indexes as indexes import workbench_target_state as state import workbench_feedback as feedback_module import workbench_db as workbench +import workbench_scan_start as scan_start +import deep_scan_workbench as deep_workbench from workbench_feedback import get_scan_feedback from workbench_schema import MIGRATIONS, apply_migrations @@ -53,6 +56,7 @@ missing = set() resolution_errors = {} probes = Counter() origins = Counter() +real_identity_details = state._repository_identity_details description = SimpleNamespace( st_mode=stat.S_IFREG, st_dev=44, st_ino=55, st_ctime_ns=66 ) @@ -81,7 +85,11 @@ def add_target(name, stored, live=None, relative=".", birth=1_000_000_000): return details[path] -def add_scan(scan_id, target, owner="current", started=timestamp, created=timestamp): +current_generation = object() + + +def add_scan(scan_id, target, owner="current", started=timestamp, created=timestamp, + generation=current_generation, status="complete"): path = paths[target] device, inode = metadata[path].st_dev, metadata[path].st_ino if owner == "missing": @@ -93,14 +101,26 @@ def add_scan(scan_id, target, owner="current", started=timestamp, created=timest "VALUES (?, ?, ?, ?, ?)", ("workspace-" + scan_id, path, target, timestamp, timestamp), ) + supports_generation = any( + row["name"] == "repository_generation" + for row in connection.execute("PRAGMA table_info(scans)") + ) + if generation is current_generation: + generation = connection.execute( + "SELECT repository_identity FROM security_targets WHERE id = ?", (target,) + ).fetchone()[0] if state.supports_repository_identity(connection) else None + include_generation = supports_generation and generation is not None + generation_column = ", repository_generation" if include_generation else "" + generation_placeholder = ", ?" if include_generation else "" connection.execute( "INSERT INTO scans (id, workspace_id, target_path, target_id, target_device, " "target_inode, target_revision, scope, mode, scan_dir, status, phase, " - "started_at, completed_at, created_at, updated_at) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + f"started_at, completed_at, created_at, updated_at{generation_column}) " + f"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?{generation_placeholder})", (scan_id, "workspace-" + scan_id, path, target, device, inode, "synthetic", - ".", "standard", str(root / "scans" / scan_id), "complete", "reporting", - started, created, created, created), + ".", "standard", str(root / "scans" / scan_id), status, "reporting", + started, created if status == "complete" else None, created, created, + *((generation,) if include_generation else ())), ) connection.execute( "INSERT INTO scan_progress (scan_id, updated_at) VALUES (?, ?)", @@ -150,7 +170,7 @@ def resolve_path(path, *args, **kwargs): def stat_path(path, *args, **kwargs): value = str(path) if value in missing: - raise FileNotFoundError(value) + raise FileNotFoundError(errno.ENOENT, "Synthetic missing path", value) return metadata[value] if value in metadata else original_stat(path, *args, **kwargs) @@ -203,7 +223,169 @@ with ExitStack() as stack: stack.enter_context(patch.object(Path, "lstat", lstat_path)) stack.enter_context(patch.object(state, "_repository_identity_details", identity_details)) stack.enter_context(patch.object(state, "repository_origin", origin)) - if scenario == "birth-time": + if scenario == "generation-metadata": + main = root / "main\ncheckout" + linked = root / "linked\ncheckout" + common = main / ".git" + admin = common / "worktrees" / "linked" + objects = common / "objects" + active = {"root": main, "gitdir": common, "nul": False} + commands = [] + directory = lambda inode, birth: SimpleNamespace( + st_mode=stat.S_IFDIR, st_dev=7, st_ino=inode, st_birthtime_ns=birth + ) + records = { + str(common): directory(20, 100), str(objects): directory(21, 200), + str(admin): directory(22, 300), + str(linked / ".git"): SimpleNamespace(st_mode=stat.S_IFREG), + } + files = { + str(linked / ".git"): b"gitdir: " + os.fsencode(admin) + b"\n", + str(admin / "gitdir"): os.fsencode(linked / ".git") + b"\n", + } + def git_bytes(target, *arguments): + commands.append(arguments) + if arguments == ("worktree", "list", "--porcelain", "-z"): + return b"worktree " + os.fsencode(active["root"]) + b"\0\0" if active["nul"] else None + paths_by_command = { + ("rev-parse", "--show-toplevel"): active["root"], + ("rev-parse", "--path-format=absolute", "--git-common-dir"): common, + ("rev-parse", "--absolute-git-dir"): active["gitdir"], + ("rev-parse", "--path-format=absolute", "--git-path", "objects"): objects, + } + value = paths_by_command.get(arguments) + return os.fsencode(value) + b"\n" if value is not None else None + with patch.object(state, "git_bytes", git_bytes), \ + patch.object(os.path, "realpath", side_effect=os.path.abspath), \ + patch.object(Path, "stat", lambda path, *a, **k: records[str(path)]), \ + patch.object(Path, "lstat", lambda path, *a, **k: records[str(path)]), \ + patch.object(Path, "read_bytes", lambda path: files[str(path)]), \ + patch.object(state, "_repository_birth_time_ns", side_effect=lambda path, value: value.st_birthtime_ns): + first = real_identity_details(main) + same = real_identity_details(main) + active.update(root=linked, gitdir=admin) + old_linked = real_identity_details(linked) + files[str(admin / "gitdir")] = os.fsencode(main / ".git") + b"\n" + wrong_backlink = real_identity_details(linked) + files[str(admin / "gitdir")] = os.fsencode(linked / ".git") + b"\n" + active["nul"] = True + nul_linked = real_identity_details(linked) + records[str(objects)] = directory(23, 400) + changed = real_identity_details(linked) + print(json.dumps({ + "oldMain": first is not None, + "oldLinkedMatches": old_linked.value == first.value, + "nulMatches": nul_linked.value == first.value, + "wrongBacklink": wrong_backlink, + "unchanged": same.value == first.value, + "objectInstanceChangesGeneration": changed.value != first.value, + "legacyMaterialUnchanged": changed.legacy_value == first.legacy_value, + "domainSeparated": first.value != first.legacy_value, + "actualObjectLookup": ("rev-parse", "--path-format=absolute", "--git-path", "objects") in commands, + })) + elif scenario == "scan-generation": + for name, value in (("requested", "generation-current"), ("alias", "generation-current"), + ("clone", "generation-clone")): + add_target(name, value) + add_scan("requested-new", "requested") + add_scan("alias-proved", "alias") + add_scan("alias-old-client", "alias", generation=None) + add_scan("requested-legacy", "requested", generation=None) + add_scan("clone-scan", "clone") + proved = add_finding("alias-proved", "proved-review", closed=True) + ignored = add_finding("alias-old-client", "unproved-review", closed=True) + current = add_finding("requested-new", "current-open") + add_finding("requested-legacy", "local-review", closed=True) + connection.execute("INSERT INTO scan_comparisons VALUES (?, ?, ?, ?, ?)", + ("alias-old-client", "requested-new", "{}", timestamp, timestamp)) + connection.execute("INSERT INTO scan_comparison_matches VALUES (?, ?, ?, ?, ?)", + ("alias-old-client", "requested-new", ignored, current, "Synthetic match")) + cache = state.RepositoryIdentityCache(connection) + scope = cache.scope("requested") + clause, values = scope.sql() + selected = sorted(row[0] for row in connection.execute(f"SELECT id FROM scans WHERE {clause}", values)) + feedback = get_scan_feedback(connection, connection.execute("SELECT * FROM scans WHERE id = 'requested-new'").fetchone()) + indexed = findings("requested") + with patch.object(state, "repository_origin", return_value=("example.test", "same-repository")): + rows = {row["id"]: history._scan_with_repository_identity(connection, row) for row in connection.execute("SELECT * FROM scans")} + explicit_clones = history._same_repository(connection, rows["requested-new"], rows["clone-scan"]) + explicit_legacy = history._same_repository(connection, rows["alias-old-client"], rows["clone-scan"]) + contradictory = dict(rows["clone-scan"], repository_generation="different-snapshot") + own_contradiction = history._same_repository(connection, rows["requested-new"], contradictory) + legacy = add_target("weak", None, "generation-strong") + connection.execute("UPDATE security_targets SET repository_identity = ? WHERE id = 'weak'", (legacy.legacy_value,)) + add_scan("weak-history", "weak", generation=None) + registration = state.register_security_target(connection, paths["weak"]) + add_finding("requested-new", "shared-decision", closed=True) + add_finding("alias-proved", "shared-decision") + missing.add(paths["alias"]) + absent_findings = sum(row["status"] == "open" for row in findings("alias")) + absent_repository = next(row for row in indexes.list_repositories(connection)["repositories"] if row["targetId"] == "alias") + print(json.dumps({ + "selected": selected, + "predicate": {"binds": len(values), "ors": clause.count(" OR ")}, + "feedback": sorted(row["findingId"] for row in feedback["falsePositives"]), + "findings": sorted((row["findingId"], row["occurrenceCount"], row["status"]) for row in indexed), + "oldClientGeneration": rows["alias-old-client"]["repository_generation"], + "explicitClones": explicit_clones, "explicitLegacy": explicit_legacy, + "ownContradiction": own_contradiction, + "weakRegistration": registration.repository_generation, + "weakHistory": connection.execute("SELECT repository_generation FROM scans WHERE id = 'weak-history'").fetchone()[0], + "weakBindingPreserved": connection.execute("SELECT repository_identity FROM security_targets WHERE id = 'weak'").fetchone()[0] == legacy.legacy_value, + "absentExactCounts": [absent_findings, absent_repository["openFindingsCount"]], + })) + elif scenario == "scan-writers": + add_target("writer", "generation-writer") + target = Path(paths["writer"]) + connection.execute("INSERT INTO workspaces (id, target_id, target_path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)", + ("current-workspace", "writer", str(target), timestamp, timestamp)) + connection.commit() + checks = [] + def register(database, path): + checks.append(database.in_transaction) + return state.register_security_target(database, path) + connection.execute("BEGIN IMMEDIATE") + registration = register(connection, str(target)) + scan_start.insert_running_scan( + connection, scan_id="current-writer", workspace=connection.execute("SELECT * FROM workspaces WHERE id = 'current-workspace'").fetchone(), + target=target, scope=".", diff_target=None, + target_identity=("synthetic", None, 7, metadata[str(target)].st_ino), + repository_generation=registration.repository_generation, + target_root=root / "artifacts", target_summary=None, scope_file_count=0, + timestamp=timestamp, scan_dir=root / "artifacts" / "current", + ) + connection.commit() + add_scan("old-writer", "writer", generation=None) + connection.commit() + with ExitStack() as mocks: + for name, value in { + "require_target": target, "require_remediation_target": target, + "require_scope": ".", "existing_deep_scan_for_target": None, + "terminal_deep_scan_for_target_snapshot": None, "git_revision": "synthetic", + "worktree_content_digest": "snapshot", "directory_snapshot_regular_file_count": 0, + "effective_deep_scan_config": {}, "now": timestamp, + "compact_timestamp": "synthetic", "state_dir": root, + }.items(): + mocks.enter_context(patch.object(deep_workbench, name, return_value=value)) + mocks.enter_context(patch.object(deep_workbench, "require_scannable_target")) + mocks.enter_context(patch.object(deep_workbench, "safe_segment", side_effect=lambda value: value)) + mocks.enter_context(patch.object(deep_workbench, "register_security_target", side_effect=register)) + mocks.enter_context(patch.object(deep_workbench, "require_scan", side_effect=lambda database, scan_id: database.execute("SELECT * FROM scans WHERE id = ?", (scan_id,)).fetchone())) + mocks.enter_context(patch.object(deep_workbench, "ensure_deep_scan_run")) + mocks.enter_context(patch.object(deep_workbench, "deep_scan_result", side_effect=lambda database, scan_id, **kwargs: {"scanId": scan_id})) + mocks.enter_context(patch.object(Path, "mkdir")) + mocks.enter_context(patch.object(Path, "resolve", lambda path, *a, **k: path)) + mocks.enter_context(patch.object(deep_workbench.tempfile, "mkdtemp", return_value=str(root / "artifacts" / "deep"))) + deep = deep_workbench.begin_deep_scan_for_target(connection, argparse.Namespace( + target_path=str(target), scope=".", workflow_version="synthetic", scan_root=str(root / "artifacts"), + user_context=None, model=None, reasoning_effort=None, + ), "synthetic-thread") + print(json.dumps({ + "generations": {row["id"]: row["repository_generation"] for row in connection.execute("SELECT id, repository_generation FROM scans")}, + "deepId": deep["scanId"], "registeredInsideTransaction": checks, + "generationIndex": connection.execute("SELECT 1 FROM sqlite_master WHERE name = 'scans_by_repository_generation'").fetchone() is not None, + })) + elif scenario == "birth-time": path = "/synthetic repository/.git" real_sizeof = ctypes.sizeof @@ -346,7 +528,10 @@ with ExitStack() as stack: feedback_scope = [] feedback_queries = [] def scoped_index(database, **kwargs): - feedback_scope.extend(sorted(kwargs["target_ids"])) + clause, values = kwargs["scan_scope"].sql() + feedback_scope.extend(sorted(row[0] for row in database.execute( + f"SELECT DISTINCT target_id FROM scans WHERE {clause}", values + ))) return indexes._indexed_findings(database, **kwargs) connection.set_trace_callback(feedback_queries.append) with patch.object(feedback_module, "_indexed_findings", side_effect=scoped_index): @@ -369,7 +554,7 @@ with ExitStack() as stack: "feedback": [row["findingId"] for row in feedback["falsePositives"]], "feedbackScope": feedback_scope, "feedbackIndexQueriesScoped": len(index_queries) == 3 and all( - "target_id IN (" in query for query in index_queries + "repository_generation = " in query for query in index_queries ), "listingRequestedProbes": listing_probes.get(paths["requested"], 0), "matchingRequestedProbes": matching_probes.get(paths["requested"], 0), @@ -391,13 +576,9 @@ with ExitStack() as stack: add_scan("legacy-a", "first", started="2026-07-31T02:00:00Z", created="2026-07-31T04:00:00Z") add_scan("legacy-missing", "first", started="2026-07-31T03:00:00Z") connection.execute("UPDATE scans SET completed_at = NULL WHERE id = 'legacy-missing'") - add_scan("visible-last", "first", started="2026-08-01T01:00:00Z") - add_scan("visible-first", "second", started="2026-08-01T02:00:00Z") - connection.execute( - "UPDATE scans SET status = 'running', completed_at = NULL " - "WHERE id IN ('visible-last', 'visible-first')" - ) apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) + add_scan("visible-last", "first", started="2026-08-01T01:00:00Z", status="running") + add_scan("visible-first", "second", started="2026-08-01T02:00:00Z", status="running") def sequences(): return {row["id"]: row["completion_sequence"] for row in connection.execute( "SELECT id, completion_sequence FROM scans ORDER BY completion_sequence, id" @@ -433,6 +614,9 @@ with ExitStack() as stack: apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) print(json.dumps({ "legacy": legacy, + "legacyGenerationsNull": all(row[0] is None for row in connection.execute( + "SELECT repository_generation FROM scans WHERE id LIKE 'legacy-%'" + )), "firstPredecessors": first_predecessors, "lastPredecessors": last_predecessors, "reciprocalPredecessors": reciprocal_predecessors, @@ -538,11 +722,11 @@ with ExitStack() as stack: stack.enter_context(patch.object(workbench, "scan_target_identity", return_value=("synthetic", None, 7, metadata[str(target)].st_ino))) stack.enter_context(patch.object(workbench, "scan_target_root", return_value=target_root)) checks = [] - original_ensure = workbench.ensure_security_target - def ensure(database, target_path): + original_register = workbench.register_security_target + def register(database, target_path): checks.append({"transaction": database.in_transaction, "path": target_path}) - return original_ensure(database, target_path) - stack.enter_context(patch.object(workbench, "ensure_security_target", side_effect=ensure)) + return original_register(database, target_path) + stack.enter_context(patch.object(workbench, "register_security_target", side_effect=register)) class StartAccepted(Exception): pass stack.enter_context(patch.object(workbench, "insert_running_scan", side_effect=StartAccepted)) @@ -816,10 +1000,25 @@ with ExitStack() as stack: accepted.append(compared["afterScanId"] == after and saved["beforeScanId"] == before) conflicting = history._same_repository( connection, - {"target_id": "same-target", "repository_identity": "repository-first"}, - {"target_id": "same-target", "repository_identity": "repository-second"}, + {"target_id": "same-target", "repository_generation": "repository-first"}, + {"target_id": "same-target", "repository_generation": "repository-second"}, identities=Mock(), ) + targetless = {"target_id": None, "target_path": paths["legacy"]} + healthy = state.RepositoryTargetState( + "", paths["legacy"], None, resolved_path=paths["legacy"], ownership_matches=True + ) + refused_targetless = state.RepositoryTargetState( + "", paths["legacy"], None, resolved_path=paths["legacy"] + ) + healthy_path = history._same_repository( + connection, targetless, targetless, + identities=Mock(for_row=Mock(return_value=healthy)), + ) + refused_path = history._same_repository( + connection, targetless, targetless, + identities=Mock(for_row=Mock(return_value=refused_targetless)), + ) add_target("unregistered", None) connection.execute("DELETE FROM security_targets WHERE id = 'unregistered'") refused = state.RepositoryTargetState("", paths["unregistered"], None) @@ -837,6 +1036,7 @@ with ExitStack() as stack: automatic_rejected = False print(json.dumps({ "accepted": accepted, "conflictingIdentitiesAccepted": conflicting, + "healthyTargetless": healthy_path, "refusedTargetless": refused_path, "savedCount": connection.execute("SELECT COUNT(*) FROM scan_comparisons").fetchone()[0], "unregisteredAutomaticRequesterRejected": automatic_rejected, })) @@ -866,9 +1066,11 @@ with ExitStack() as stack: add_scan(name + "-scan", name, owner, "invalid" if name == "invalid-time" else timestamp) if name == "unavailable": missing.add(paths[name]) - expected[name] = identity.value if name in { - "old-basic", "old-description", "current", "no-scans", "scope-upper", "scope-lower" - } else None + expected[name] = ( + identity.value if name == "current" else identity.legacy_value if name in { + "old-basic", "old-description", "no-scans", "scope-upper", "scope-lower" + } else None + ) apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) first = { row["id"]: row["repository_identity"] @@ -901,6 +1103,55 @@ function run(scenario: string): Record { return JSON.parse(execution.stdout) as Record; } +test("uses registered old-Git paths and stable object-directory generation evidence", () => { + expect(run("generation-metadata")).toEqual({ + oldMain: true, + oldLinkedMatches: true, + nulMatches: true, + wrongBacklink: null, + unchanged: true, + objectInstanceChangesGeneration: true, + legacyMaterialUnchanged: true, + domainSeparated: true, + actualObjectLookup: true, + }); +}); + +test("keeps automatic history bounded by saved scan generation", () => { + const result = run("scan-generation"); + expect(result["selected"]).toEqual([ + "alias-proved", + "requested-legacy", + "requested-new", + ]); + expect(result["predicate"]).toEqual({ binds: 2, ors: 1 }); + expect(result["feedback"]).toEqual(["local-review", "proved-review"]); + expect(result["findings"]).toEqual([ + ["current-open", 1, "open"], + ["local-review", 1, "closed"], + ["proved-review", 1, "closed"], + ]); + expect(result["oldClientGeneration"]).toBeNull(); + expect(result["explicitClones"]).toBe(true); + expect(result["explicitLegacy"]).toBe(true); + expect(result["ownContradiction"]).toBe(false); + expect(result["weakRegistration"]).toBe("generation-strong"); + expect(result["weakHistory"]).toBeNull(); + expect(result["weakBindingPreserved"]).toBe(true); + expect(result["absentExactCounts"]).toEqual([1, 1]); +}); + +test("records generation explicitly in both transactional parent scan writers", () => { + const result = run("scan-writers"); + expect(result["generations"]).toEqual({ + "current-writer": "generation-writer", + "old-writer": null, + [result["deepId"] as string]: "generation-writer", + }); + expect(result["registeredInsideTransaction"]).toEqual([true, true]); + expect(result["generationIndex"]).toBe(true); +}); + test("reads precise Linux birth time through supported native interfaces", () => { const result = run("birth-time"); const cases = result["cases"] as Record; @@ -973,8 +1224,8 @@ test("reuses established aliases and probes each saved target once per request", expect(result["feedbackIndexQueriesScoped"]).toBe(true); expect(result["listingRequestedProbes"]).toBe(1); expect(result["matchingRequestedProbes"]).toBe(1); - expect(result["matchingUnrelatedProbes"]).toBe(1); - expect(result["matchingUnrelatedOrigins"]).toBe(1); + expect(result["matchingUnrelatedProbes"]).toBe(0); + expect(result["matchingUnrelatedOrigins"]).toBe(0); expect(result["indexingMaxProbes"]).toBe(1); expect(result["feedbackMaxProbes"]).toBe(1); expect(result["legacyStored"]).toBe("repository-current"); @@ -991,6 +1242,7 @@ test("orders completed history by database visibility across legacy and current "legacy-a": 2, "legacy-b": 3, }); + expect(result["legacyGenerationsNull"]).toBe(true); expect(result["firstPredecessors"]).not.toContain("visible-last"); expect(result["lastPredecessors"]).toContain("visible-first"); expect(result["reciprocalPredecessors"]).not.toContain("visible-last"); @@ -1136,6 +1388,8 @@ test("compares sealed history from persisted evidence without weakening automati expect(result["accepted"]).toEqual([true, true]); expect(result["conflictingIdentitiesAccepted"]).toBe(false); + expect(result["healthyTargetless"]).toBe(true); + expect(result["refusedTargetless"]).toBe(false); expect(result["savedCount"]).toBe(2); expect(result["unregisteredAutomaticRequesterRejected"]).toBe(true); }); @@ -1183,11 +1437,8 @@ test("quarantines unproved public-v30 bindings without discarding historical rec }); expect(result["recordsPreserved"]).toBe(true); expect(result["targetIdsPreserved"]).toBe(true); - expect(result["aliases"]).toEqual(["current-owner", "removed-valid"]); + expect(result["aliases"]).toEqual(["current-owner"]); expect(result["removedExact"]).toEqual(["removed-old-scan"]); - expect(result["visibleFindings"]).toEqual([ - "current-owner-finding", - "removed-valid-finding", - ]); + expect(result["visibleFindings"]).toEqual(["current-owner-finding"]); expect(result["oldRegistrationRejected"]).toBe(true); }); diff --git a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts index dcffa8eaf..209e8dec2 100644 --- a/sdk/typescript/tests-ts/workbench-repository-identity.test.ts +++ b/sdk/typescript/tests-ts/workbench-repository-identity.test.ts @@ -91,6 +91,7 @@ from workbench_target_state import ( backfill_repository_identities, backfill_security_targets, ensure_security_target, + register_security_target, repository_identity, repository_relative_path, stable_target_id, @@ -122,9 +123,10 @@ def git(target, *args): def add_scan(scan_id, target, ownership="current", verify_ownership=True): target = Path(target) - target_id = ensure_security_target( + registration = register_security_target( connection, str(target), verify_ownership=verify_ownership ) + target_id = registration.target_id metadata = target.stat() device = serialize_filesystem_identity(metadata.st_dev) inode = serialize_filesystem_identity(metadata.st_ino) @@ -142,12 +144,12 @@ def add_scan(scan_id, target, ownership="current", verify_ownership=True): (workspace_id, str(target), target_id, timestamp, timestamp), ) connection.execute( - "INSERT INTO scans (id, workspace_id, target_path, target_id, target_device, " + "INSERT INTO scans (id, workspace_id, target_path, target_id, repository_generation, target_device, " "target_inode, target_revision, scope, mode, scan_dir, status, phase, " "started_at, created_at, updated_at) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( - scan_id, workspace_id, str(target), target_id, device, inode, + scan_id, workspace_id, str(target), target_id, registration.repository_generation, device, inode, "synthetic-revision", ".", "standard", str(root / "scans" / scan_id), "complete", "reporting", timestamp, timestamp, timestamp, ), @@ -230,7 +232,7 @@ if scenario == "identity": custom_template.mkdir() git(custom_template, "init", "-q", "--template=") markerless_repository_identity = repository_identity(custom_template) - with patch("workbench_target_state._repository_birth_time_ns", side_effect=(41, 42)): + with patch("workbench_target_state._repository_birth_time_ns", side_effect=(41, 100, 42, 100)): recycled_inode_distinguished = ( repository_identity(repository) != repository_identity(repository) ) @@ -606,6 +608,9 @@ elif scenario == "candidate-generation": "UPDATE security_targets SET repository_identity = NULL WHERE id = ?", (legacy_id,), ) + connection.execute( + "UPDATE scans SET repository_generation = NULL WHERE id = ?", ("trusted-alias",) + ) verified_legacy = automatic_history() connection.execute( "UPDATE scans SET target_device = NULL, target_inode = NULL WHERE id = ?", @@ -779,8 +784,8 @@ describe("durable workbench repository identities", () => { expect(before["serviceB"]).toEqual(["canonical-b"]); expect(after["root"]).toEqual(["canonical-root", "linked-root"]); expect(after["serviceA"]).toEqual(["canonical-a", "linked-a"]); - expect(result["matchedScanCount"]).toBe(3); - expect(result["matchingBatches"]).toBe(2); + expect(result["matchedScanCount"]).toBe(2); + expect(result["matchingBatches"]).toBe(1); expect(result["unavailableScans"]).toBe(1); expect(result["compared"]).toBe(true); expect(result["saved"]).toBe(true); @@ -887,7 +892,7 @@ describe("durable workbench repository identities", () => { const result = runProbe("unverified-owner", fixture()); expect(result["before"]).toEqual(["trusted-alias"]); - expect(result["after"]).toEqual(["unverified-owner"]); + expect(result["after"]).toEqual([]); }, 30_000); test("keeps non-Git paths isolated and never equates unrelated null target IDs", () => { @@ -896,8 +901,8 @@ describe("durable workbench repository identities", () => { expect(result["firstIdentity"]).toBeNull(); expect(result["secondIdentity"]).toBeNull(); expect(result["differentNullIdsMatch"]).toBe(false); - expect(result["sameNullPathMatches"]).toBe(true); - expect(result["firstScans"]).toEqual(["legacy-a", "plain-a"]); + expect(result["sameNullPathMatches"]).toBe(false); + expect(result["firstScans"]).toEqual([]); expect(result["secondScans"]).toEqual(["plain-b"]); }, 30_000); }); From 6f2f60b7346d68d51bbce515ab6b15cfe82ab767 Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sun, 16 Aug 2026 08:37:43 -0700 Subject: [PATCH 15/21] Preserve archived scans across registration failures --- sdk/typescript/README.md | 9 +- .../_bundled_plugin/scripts/workbench_cli.py | 7 + .../_bundled_plugin/scripts/workbench_db.py | 11 +- .../scripts/workbench_native_indexes.py | 38 +-- .../scripts/workbench_scan_start.py | 51 ++++ sdk/typescript/src/api.ts | 118 +++++++- sdk/typescript/tests-ts/api.test.ts | 148 +++++++++- .../repository-feedback-aliases.test.ts | 41 ++- .../tests-ts/repository-findings.test.ts | 42 ++- .../workbench-canonical-paths.test.ts | 14 +- .../tests-ts/workbench-identity-cache.test.ts | 260 ++++++++++++++++++ .../tests-ts/workbench-scan-history.test.ts | 11 +- 12 files changed, 655 insertions(+), 95 deletions(-) diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 8da806da1..aa594c501 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -732,9 +732,12 @@ the same reason still applies. Pass `SCAN_ID` to rerun another scan. `scans match BEFORE_SCAN_ID AFTER_SCAN_ID` links findings with the same root -cause; `scans match --all` matches all completed scans of the current repository, -including other worktrees and clones. Saved matches appear in `scans show` and -are reused unless `--force` is passed. Scans without sealed artifacts are skipped. +cause. `scans match --all` matches eligible completed scans in the current +repository generation, including linked worktrees; legacy history without a +saved generation stays target-local. Explicit comparisons between separately +verified clones remain available without sharing their history. Saved matches +appear in `scans show` and are reused unless `--force` is passed. Scans without +sealed artifacts are skipped. `scans compare` compares the two latest completed scans. Pass one scan ID to compare it with the latest completed scan, or two IDs to select both scans. It diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py b/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py index fa508f2a8..82540ade1 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py @@ -148,6 +148,13 @@ def parse_args(description: str) -> argparse.Namespace: register_cli_scan.add_argument("--archive-existing", action="store_true") register_cli_scan.add_argument("--archived-scan-dir") + restore_cli_scan_archive = subparsers.add_parser("restore-cli-scan-archive") + restore_cli_scan_archive.add_argument("--scan-dir", required=True) + restore_cli_scan_archive.add_argument("--archived-scan-dir", required=True) + previous_scan = restore_cli_scan_archive.add_mutually_exclusive_group(required=True) + previous_scan.add_argument("--previous-scan-id") + previous_scan.add_argument("--previous-scan-absent", action="store_true") + set_scan_thread = subparsers.add_parser("set-scan-thread") set_scan_thread.add_argument("--scan-id", required=True) set_scan_thread.add_argument("--thread-id", required=True) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index 49733c11e..4ce1af128 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -83,6 +83,7 @@ archive_scan, compact_timestamp, insert_running_scan, + restore_cli_scan_archive, safe_segment, scan_diff_identity, scan_target_identity, @@ -1660,7 +1661,6 @@ def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace) connection.execute("BEGIN IMMEDIATE") try: - archive_scan(connection, args, scan_dir, timestamp, require_canonical_scan_directory) registration = register_security_target(connection, str(repository)) target_id = registration.target_id if parent_scan_id is not None: @@ -1670,6 +1670,11 @@ def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace) ).contains(parent): raise SystemExit("A rerun must belong to the same repository as its parent scan.") + scan_dir = require_canonical_scan_directory(scan_dir) + if next(scan_dir.iterdir(), None) is not None: + raise SystemExit("The scan artifact directory must be empty before the scan starts.") + archive_scan(connection, args, scan_dir, timestamp, require_canonical_scan_directory) + connection.execute( """ INSERT INTO workspaces ( @@ -3905,6 +3910,10 @@ def main() -> None: ) elif args.command == "register-cli-scan": result = register_cli_scan(connection, args) + elif args.command == "restore-cli-scan-archive": + result = restore_cli_scan_archive( + connection, args, require_canonical_scan_directory + ) elif args.command == "set-scan-thread": result = set_scan_thread(connection, args) elif args.command == "get-scan-recipe": diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py index cf67b5c5e..693eccf2f 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py @@ -180,24 +180,25 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: completed_at_column = ( "scans.completed_at" if "completed_at" in identities.scan_columns else "NULL" ) + completed_scans = sorted( + connection.execute( + f""" + SELECT scans.target_id, scans.id, scans.started_at, + {generation_sql("scans")} AS repository_generation, + {completed_at_column} AS completed_at, + {completion_column} AS completion_sequence + FROM scans + JOIN security_targets AS targets ON targets.id = scans.target_id + WHERE scans.status = 'complete' AND {target_filter} + """, + target_values, + ), + key=scan_history._scan_completion_order, + ) latest_scan_by_repository = { - scan_repository_group(row): row["id"] - for row in sorted( - connection.execute( - f""" - SELECT scans.target_id, scans.id, scans.started_at, - {generation_sql("scans")} AS repository_generation, - {completed_at_column} AS completed_at, - {completion_column} AS completion_sequence - FROM scans - JOIN security_targets AS targets ON targets.id = scans.target_id - WHERE scans.status = 'complete' AND {target_filter} - """, - target_values, - ), - key=scan_history._scan_completion_order, - ) + scan_repository_group(row): row["id"] for row in completed_scans } + selected_latest_scan_id = completed_scans[-1]["id"] if completed_scans else None grouped: dict[tuple[tuple[str, str], str], list[sqlite3.Row]] = {} for row in connection.execute( @@ -262,7 +263,10 @@ def group(identity: tuple[tuple[str, str], str]) -> tuple[tuple[str, str], str]: ): status = "open" scans = sorted({(row["scan_started_at"], row["scan_id"]) for row in occurrences}) - latest_scan_id = latest_scan_by_repository.get(scan_repository_group(latest)) + latest_scan_id = ( + selected_latest_scan_id if scan_scope is not None + else latest_scan_by_repository.get(scan_repository_group(latest)) + ) findings.append( { **dict(latest), diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py index deffa8789..7d4cedb04 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_start.py @@ -148,6 +148,57 @@ def archive_scan( ) +def restore_cli_scan_archive( + connection: sqlite3.Connection, + args: argparse.Namespace, + canonical_directory: Callable[[Path], Path], +) -> dict[str, str]: + supplied_scan_dir = Path(args.scan_dir).expanduser().absolute() + supplied_archive = Path(args.archived_scan_dir).expanduser().absolute() + connection.execute("BEGIN IMMEDIATE") + try: + scan_dir = supplied_scan_dir.resolve() + archived_scan_dir = supplied_archive.resolve() + archived_owner = connection.execute( + "SELECT id FROM scans WHERE scan_dir = ?", (str(archived_scan_dir),) + ).fetchone() + previous_scan = connection.execute( + "SELECT id FROM scans WHERE scan_dir = ?", (str(scan_dir),) + ).fetchone() + previous_scan_id = previous_scan["id"] if previous_scan is not None else None + if archived_owner is not None: + disposition = "already-recorded" + elif previous_scan_id != args.previous_scan_id: + disposition = "ownership-changed" + else: + archived_scan_dir = canonical_directory(supplied_archive) + if ( + os.path.normcase(supplied_scan_dir) != os.path.normcase(scan_dir) + or archived_scan_dir.parent != scan_dir.parent + or not archived_scan_dir.name.startswith(f"{scan_dir.name}.previous-") + ): + raise SystemExit( + "The archived scan must be a previous sibling of the scan directory." + ) + try: + supplied_scan_dir.lstat() + except FileNotFoundError: + pass + else: + canonical_directory(supplied_scan_dir).rmdir() + archived_scan_dir.rename(scan_dir) + disposition = "restored" + connection.commit() + except BaseException: + connection.rollback() + raise + return { + "disposition": disposition, + "scanDir": str(scan_dir), + "archivedScanDir": str(archived_scan_dir), + } + + def insert_running_scan( connection: sqlite3.Connection, *, diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts index d17d83a71..0fad45240 100644 --- a/sdk/typescript/src/api.ts +++ b/sdk/typescript/src/api.ts @@ -423,6 +423,14 @@ export class CodexSecurity { ]); let scanDir = ""; let archivedScanDir: string | null = null; + const archiveRecovery: { + pending: { + scanDir: string; + archivedScanDir: string; + previousScanId: string | null; + options: WorkbenchCommandOptions; + } | null; + } = { pending: null }; let targetPathsFile: string | null = null; let knowledgeBase: PreparedKnowledgeBase | null = null; let costTracker: ScanCostTracker | null = null; @@ -638,6 +646,52 @@ export class CodexSecurity { signal, }); checkOpen(); + const workbenchOptions: WorkbenchCommandOptions = { + python, + pluginRoot: runtime.plugin.pluginRoot, + environment: { + ...selectedScanEnvironment( + runtime.environment, + options.auth, + modelProvider, + ), + CODEX_SECURITY_STATE_DIR: stateDirectory, + }, + signal, + failureMessage: "Could not save the Codex Security scan", + }; + let previousOutput: { scanId: string | null } | null = null; + if (requestedOutput !== null && options.archiveExisting === true) { + const history = await workbench( + { + ...workbenchOptions, + failureMessage: + "Could not inspect the previous Codex Security scan", + }, + ["list-scans", "--scan-root", requestedOutput], + ); + const scans = history["scans"]; + const previous = Array.isArray(scans) + ? scans.filter( + (scan): scan is JsonObject => + isRecord(scan) && scan["scanDir"] === requestedOutput, + ) + : null; + const previousScanId = previous?.[0]?.["scanId"]; + if ( + previous === null || + previous.length > 1 || + (previous.length === 1 && + (typeof previousScanId !== "string" || previousScanId.length === 0)) + ) { + throw new CodexSecurityError( + "The Codex Security workbench returned invalid previous scan history.", + ); + } + previousOutput = { + scanId: typeof previousScanId === "string" ? previousScanId : null, + }; + } const scanOutputRoot = requestedOutput === null && this.#dependencies.prepareOutputDir === undefined @@ -658,6 +712,14 @@ export class CodexSecurity { options.archiveExisting, (archiveDir) => { archivedScanDir = archiveDir; + if (requestedOutput !== null && previousOutput !== null) { + archiveRecovery.pending = { + scanDir: requestedOutput, + archivedScanDir: archiveDir, + previousScanId: previousOutput.scanId, + options: workbenchOptions, + }; + } notifyObserver( "onOutputArchived", options.onOutputArchived, @@ -810,20 +872,6 @@ export class CodexSecurity { options.maxCostUsd, deepScanOptions(options), ); - const workbenchOptions: WorkbenchCommandOptions = { - python, - pluginRoot: runtime.plugin.pluginRoot, - environment: { - ...selectedScanEnvironment( - runtime.environment, - options.auth, - modelProvider, - ), - CODEX_SECURITY_STATE_DIR: stateDirectory, - }, - signal, - failureMessage: "Could not save the Codex Security scan", - }; const registration = await workbench(workbenchOptions, [ "register-cli-scan", "--repository", @@ -840,6 +888,7 @@ export class CodexSecurity { ? [] : ["--parent-scan-id", options.parentScanId]), ]); + archiveRecovery.pending = null; const scanId = registration["scanId"]; const targetId = registration["targetId"]; const contract = registration["contract"]; @@ -1331,6 +1380,47 @@ export class CodexSecurity { signal.reason instanceof ScanCostLimitExceededError ? signal.reason : error; + if (archiveRecovery.pending !== null) { + const archive = archiveRecovery.pending; + archiveRecovery.pending = null; + try { + const recovery = await workbench( + { + ...archive.options, + signal: undefined, + failureMessage: + "Could not restore the previous Codex Security scan", + }, + [ + "restore-cli-scan-archive", + "--scan-dir", + archive.scanDir, + "--archived-scan-dir", + archive.archivedScanDir, + ...(archive.previousScanId === null + ? ["--previous-scan-absent"] + : ["--previous-scan-id", archive.previousScanId]), + ], + ); + if ( + recovery["scanDir"] !== archive.scanDir || + recovery["archivedScanDir"] !== archive.archivedScanDir || + (recovery["disposition"] !== "restored" && + recovery["disposition"] !== "already-recorded") + ) { + throw new CodexSecurityError( + `Previous scan output remains archived at ${archive.archivedScanDir}; its ownership could not be reconciled.`, + ); + } + } catch (recoveryError) { + notifyObserver( + "onWarning", + options.onWarning, + options.onObserverError, + `Could not restore previous scan output: ${errorMessage(recoveryError)}`, + ); + } + } if ( failure instanceof ScanCostLimitExceededError && budgetRecovery !== null && diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts index a2fc5596e..4aa383142 100644 --- a/sdk/typescript/tests-ts/api.test.ts +++ b/sdk/typescript/tests-ts/api.test.ts @@ -178,6 +178,7 @@ class TestClient extends TestClientBase { } function mockWorkbench(args: readonly string[]) { + if (args[0] === "list-scans") return { scans: [] }; if (args[0] === "register-cli-scan") return mockScanRegistration(args); if (args[0] === "get-scan-feedback") { return { @@ -1435,7 +1436,7 @@ describe("CodexSecurity orchestration", () => { falsePositives: [], }; } - if (args[0] !== "register-cli-scan") return {}; + if (args[0] !== "register-cli-scan") return mockWorkbench(args); registration = args; return mockScanRegistration(args); }, @@ -1478,6 +1479,151 @@ describe("CodexSecurity orchestration", () => { await client.close(); }); + test.each([ + ["preparation failure", "prepare", "previous", "restored", false], + ["registration refusal", "register", "previous", "restored", false], + ["unrecorded previous output", "register", null, "restored", false], + [ + "lost registration acknowledgement", + "register", + "previous", + "already-recorded", + false, + ], + ["changed output owner", "register", "previous", "ownership-changed", true], + ["recovery failure", "register", "previous", "error", true], + [ + "malformed recovery acknowledgement", + "register", + "previous", + "malformed", + true, + ], + [ + "acknowledged invalid registration", + "acknowledged", + "previous", + null, + false, + ], + ["acknowledged registration", "started", "previous", null, false], + ] as const)( + "reconciles archived output through the workbench after %s", + async (_label, failureStage, previousScanId, disposition, warns) => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const codexHome = join(root, "codex-home"); + const output = join(root, "scan"); + const archive = `${output}.previous-synthetic`; + await Promise.all([ + mkdir(repository), + mkdir(codexHome), + mkdir(output, { mode: 0o700 }), + ]); + const failure = new Error("synthetic registration failure"); + const commands: string[] = []; + const warnings: string[] = []; + let recoveryArgs: readonly string[] | undefined; + let recoverySignal: AbortSignal | undefined; + const client = new TestClient( + {}, + { + environment: {}, + prepareRuntime: async () => preparedRuntime(codexHome), + resolvePluginPython: async () => "/managed/python", + repositoryRevision: async () => null, + prepareOutputDir: async ( + requested: string | undefined, + _name: string, + _root: string | undefined, + _validate: ((path: string) => void) | undefined, + archiveExisting: boolean, + onArchived: ((path: string) => void) | undefined, + ) => { + commands.push("prepare-output"); + expect(requested).toBe(output); + expect(archiveExisting).toBe(true); + onArchived?.(archive); + if (failureStage === "prepare") throw failure; + return output; + }, + runWorkbench: async ( + workbenchOptions: { signal?: AbortSignal }, + args: readonly string[], + ) => { + commands.push(args[0]!); + if (args[0] === "list-scans") { + expect(args).toEqual(["list-scans", "--scan-root", output]); + return { + scans: [ + { scanId: "child", scanDir: join(output, "child") }, + ...(previousScanId === null + ? [] + : [{ scanId: previousScanId, scanDir: output }]), + ], + }; + } + if (args[0] === "register-cli-scan") { + if (failureStage === "acknowledged") return {}; + if (failureStage === "started") return mockScanRegistration(args); + throw failure; + } + if (args[0] === "restore-cli-scan-archive") { + recoveryArgs = args; + recoverySignal = workbenchOptions.signal; + if (disposition === "error") + throw new Error("synthetic recovery failure"); + if (disposition === "malformed") return {}; + return { disposition, scanDir: output, archivedScanDir: archive }; + } + return mockWorkbench(args); + }, + createCodex: () => { + if (failureStage === "started") throw failure; + throw new Error("model execution must not start"); + }, + }, + ); + + const operation = client.run(repository, { + outputDir: output, + archiveExisting: true, + onWarning: (warning) => warnings.push(warning), + }); + if (failureStage === "acknowledged") { + await expect(operation).rejects.toThrow("invalid scan registration"); + expect(recoveryArgs).toBeUndefined(); + } else if (failureStage === "started") { + await expect(operation).rejects.toBe(failure); + expect(recoveryArgs).toBeUndefined(); + } else { + await expect(operation).rejects.toBe(failure); + expect(recoveryArgs).toEqual([ + "restore-cli-scan-archive", + "--scan-dir", + output, + "--archived-scan-dir", + archive, + ...(previousScanId === null + ? ["--previous-scan-absent"] + : ["--previous-scan-id", previousScanId]), + ]); + expect(recoverySignal).toBeUndefined(); + } + expect(commands.slice(0, 2)).toEqual(["list-scans", "prepare-output"]); + expect(commands.includes("register-cli-scan")).toBe( + failureStage !== "prepare", + ); + await Promise.resolve(); + expect( + warnings.some((warning) => + warning.startsWith("Could not restore previous scan output:"), + ), + ).toBe(warns); + await client.close(); + }, + ); + test("reports the real scan failure when scan cleanup also fails", async () => { const root = await temporaryDirectory(); const repository = join(root, "repository"); diff --git a/sdk/typescript/tests-ts/repository-feedback-aliases.test.ts b/sdk/typescript/tests-ts/repository-feedback-aliases.test.ts index 35709a5a3..b5d8d9b59 100644 --- a/sdk/typescript/tests-ts/repository-feedback-aliases.test.ts +++ b/sdk/typescript/tests-ts/repository-feedback-aliases.test.ts @@ -6,9 +6,9 @@ import { PLUGIN_ROOT } from "./plugin-root.js"; test("reuses reviewed feedback only across matching persisted repository identities", () => { const python = (Bun.which("python3") ?? Bun.which("python"))!; const probe = ` -import json, os, sqlite3, sys, tempfile +import json, sqlite3, sys sys.path.insert(0, sys.argv[1]) -from filesystem_identity import serialize_filesystem_identity +import workbench_target_state as state from workbench_feedback import get_scan_feedback scenario = sys.argv[2] @@ -16,7 +16,7 @@ connection = sqlite3.connect(":memory:") connection.row_factory = sqlite3.Row connection.executescript(""" CREATE TABLE security_targets(id TEXT, current_path TEXT, display_name TEXT, origin TEXT); -CREATE TABLE scans(id TEXT, target_id TEXT, status TEXT, completed_at TEXT, started_at TEXT, updated_at TEXT, scope TEXT); +CREATE TABLE scans(id TEXT, target_id TEXT, target_path TEXT, repository_generation TEXT, status TEXT, completed_at TEXT, started_at TEXT, updated_at TEXT, scope TEXT); CREATE TABLE findings(id TEXT PRIMARY KEY, fingerprint TEXT, rule_id TEXT, identity_anchor TEXT, identity_instance TEXT); CREATE TABLE finding_occurrences(id TEXT, finding_id TEXT, scan_id TEXT, title TEXT, summary TEXT, severity TEXT, created_at TEXT); CREATE TABLE finding_triage(occurrence_id TEXT, status TEXT, close_reason TEXT, note TEXT, updated_at TEXT); @@ -25,6 +25,15 @@ CREATE TABLE scan_comparison_matches(before_occurrence_id TEXT, after_occurrence """) if scenario == "identities": connection.execute("ALTER TABLE security_targets ADD COLUMN repository_identity TEXT") +refused = set() +def inspect(database, target_id, path, stored, **kwargs): + repository = state.GitRepositoryIdentity(stored, ".", "synthetic", 1, 2, 3) if stored else None + return state.RepositoryTargetState( + target_id, path, stored, resolved_path=path, repository=repository, + ownership_matches=target_id not in refused, strict_owner_matches=True, + has_historical_scans=True, + ) +state._inspect_repository_target = inspect origin = "https://example.invalid/synthetic/repository" for target, identity in [ @@ -48,7 +57,8 @@ for target, identity in [ def add_scan(scan_id, target, day, status="complete"): timestamp = f"2026-03-{day:02d}T00:00:00Z" completed_at = timestamp if status == "complete" else None - connection.execute("INSERT INTO scans VALUES (?, ?, ?, ?, ?, ?, ?)", (scan_id, target, status, completed_at, timestamp, timestamp, "repository")) + generation = connection.execute("SELECT repository_identity FROM security_targets WHERE id = ?", (target,)).fetchone()[0] if scenario == "identities" else None + connection.execute("INSERT INTO scans VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", (scan_id, target, f"/{target}", generation, status, completed_at, timestamp, timestamp, "repository")) def add_finding(scan_id, finding_id, day, *, status="closed", reason="false_positive", note="Reviewed and safe"): occurrence_id = f"{scan_id}:{finding_id}" @@ -115,26 +125,9 @@ result = { "unknownSecond": feedback("unknown-second"), } if scenario == "identities": - connection.execute("ALTER TABLE scans ADD COLUMN target_path TEXT") - connection.execute("ALTER TABLE scans ADD COLUMN target_device INTEGER") - connection.execute("ALTER TABLE scans ADD COLUMN target_inode INTEGER") - with tempfile.TemporaryDirectory() as reused_path: - metadata = os.stat(reused_path) - connection.execute( - "UPDATE security_targets SET current_path = ? WHERE id = ?", - (reused_path, "primary"), - ) - connection.execute( - "UPDATE scans SET target_path = ?, target_device = ?, target_inode = ? WHERE target_id = ?", - ( - reused_path, - serialize_filesystem_identity(metadata.st_dev), - serialize_filesystem_identity(metadata.st_ino + 1), - "primary", - ), - ) - result["reusedPath"] = feedback("primary") - result["deletedAlias"] = feedback("linked") + refused.add("primary") + result["reusedPath"] = feedback("primary") + result["deletedAlias"] = feedback("linked") print(json.dumps(result)) `; diff --git a/sdk/typescript/tests-ts/repository-findings.test.ts b/sdk/typescript/tests-ts/repository-findings.test.ts index 05fce8b3a..8ed2ef007 100644 --- a/sdk/typescript/tests-ts/repository-findings.test.ts +++ b/sdk/typescript/tests-ts/repository-findings.test.ts @@ -118,16 +118,16 @@ print(json.dumps(result)) test("shares findings only between explicitly identified repository and scope aliases", () => { const python = (Bun.which("python3") ?? Bun.which("python"))!; const probe = ` -import argparse, json, os, sqlite3, sys, tempfile +import argparse, json, sqlite3, sys sys.path.insert(0, sys.argv[1]) -from filesystem_identity import serialize_filesystem_identity import workbench_native_indexes as indexes +import workbench_target_state as state connection = sqlite3.connect(":memory:") connection.row_factory = sqlite3.Row connection.executescript(""" CREATE TABLE security_targets(id TEXT, current_path TEXT, display_name TEXT, repository_identity TEXT, origin TEXT); -CREATE TABLE scans(id TEXT, target_id TEXT, scope TEXT, updated_at TEXT, status TEXT, started_at TEXT); +CREATE TABLE scans(id TEXT, target_id TEXT, target_path TEXT, repository_generation TEXT, scope TEXT, updated_at TEXT, status TEXT, started_at TEXT); CREATE TABLE finding_occurrences(id TEXT, finding_id TEXT, severity TEXT, created_at TEXT, scan_id TEXT, title TEXT, summary TEXT); CREATE TABLE finding_triage(occurrence_id TEXT, status TEXT, updated_at TEXT, close_reason TEXT); CREATE TABLE finding_locations(occurrence_id TEXT, relative_path TEXT, role TEXT, sort_order INTEGER); @@ -143,10 +143,21 @@ connection.executemany("INSERT INTO security_targets VALUES (?, ?, ?, ?, ?)", [ ("unknown-first", "/unknown-first", "Unknown first", None, origin), ("unknown-second", "/unknown-second", "Unknown second", None, origin), ]) +refused = set() +def inspect(database, target_id, path, stored, **kwargs): + repository = state.GitRepositoryIdentity(stored, ".", "synthetic", 1, 2, 3) if stored else None + return state.RepositoryTargetState( + target_id, path, stored, resolved_path=path, repository=repository, + ownership_matches=target_id not in refused, strict_owner_matches=True, + has_historical_scans=True, + ) +state._inspect_repository_target = inspect +indexes.Path.is_dir = lambda path: True def add_scan(scan_id, target, day): timestamp = f"2026-02-{day:02d}T00:00:00Z" - connection.execute("INSERT INTO scans VALUES (?, ?, ?, ?, ?, ?)", (scan_id, target, "repository", timestamp, "complete", timestamp)) + saved = connection.execute("SELECT current_path, repository_identity FROM security_targets WHERE id = ?", (target,)).fetchone() + connection.execute("INSERT INTO scans VALUES (?, ?, ?, ?, ?, ?, ?, ?)", (scan_id, target, saved["current_path"], saved["repository_identity"], "repository", timestamp, "complete", timestamp)) def add_finding(occurrence, finding, scan, severity="high"): started = connection.execute("SELECT started_at FROM scans WHERE id = ?", (scan,)).fetchone()[0] @@ -246,26 +257,9 @@ connection.execute( ("same-id-linked", "open", "2026-02-06T00:00:00Z", None), ) result["sameIdReopened"] = findings("primary") -connection.execute("ALTER TABLE scans ADD COLUMN target_path TEXT") -connection.execute("ALTER TABLE scans ADD COLUMN target_device INTEGER") -connection.execute("ALTER TABLE scans ADD COLUMN target_inode INTEGER") -with tempfile.TemporaryDirectory() as reused_path: - metadata = os.stat(reused_path) - connection.execute( - "UPDATE security_targets SET current_path = ? WHERE id = ?", - (reused_path, "primary"), - ) - connection.execute( - "UPDATE scans SET target_path = ?, target_device = ?, target_inode = ? WHERE target_id = ?", - ( - reused_path, - serialize_filesystem_identity(metadata.st_dev), - serialize_filesystem_identity(metadata.st_ino + 1), - "primary", - ), - ) - result["reusedPath"] = findings("primary") - result["deletedAlias"] = findings("linked") +refused.add("primary") +result["reusedPath"] = findings("primary") +result["deletedAlias"] = findings("linked") print(json.dumps(result)) `; diff --git a/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts b/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts index c75ae7213..fad2538ff 100644 --- a/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts +++ b/sdk/typescript/tests-ts/workbench-canonical-paths.test.ts @@ -69,11 +69,11 @@ const caseSensitiveWindowsHistoryProbe = [ "connection.row_factory = sqlite3.Row", "connection.executescript('''", "CREATE TABLE security_targets (id TEXT, current_path TEXT, repository_identity TEXT);", - "CREATE TABLE scans (id TEXT, target_id TEXT, target_path TEXT, status TEXT, started_at TEXT);", + "CREATE TABLE scans (id TEXT, target_id TEXT, target_path TEXT, repository_generation TEXT, status TEXT, started_at TEXT);", "CREATE TABLE scan_comparisons (before_scan_id TEXT, after_scan_id TEXT);", "''')", "def target(identifier, path, identity):", - " return connection.execute('SELECT ? AS target_id, ? AS target_path, ? AS repository_identity', (identifier, str(WindowsPath(path)), identity)).fetchone()", + " return connection.execute('SELECT ? AS target_id, ? AS target_path, ? AS repository_identity, ? AS repository_generation', (identifier, str(WindowsPath(path)), identity, identity)).fetchone()", "upper_root = target('upper-root', 'D:/Repository', 'root-upper')", "lower_root = target('lower-root', 'D:/repository', 'root-lower')", "upper_scope = target('upper-scope', 'D:/Repository/Service', 'scope-upper')", @@ -95,13 +95,13 @@ const caseSensitiveWindowsHistoryProbe = [ "checks = {", " 'nativeWindowsScopeEquality': WindowsPath(upper_scope['target_path']) == WindowsPath(lower_scope['target_path']),", " 'nativeWindowsRootEquality': WindowsPath(upper_root['target_path']) == WindowsPath(lower_root['target_path']),", - " 'caseSensitiveScopesMatch': cache.group(lower_scope['target_id']) == cache.group(upper_scope['target_id']),", - " 'caseSensitiveRootsMatch': cache.group(lower_root['target_id']) == cache.group(upper_root['target_id']),", - " 'linkedWorktreeMatches': cache.group(linked_scope['target_id']) == cache.group(upper_scope['target_id']),", - " 'sameOriginCloneMatches': cache.group(clone_scope['target_id']) == cache.group(upper_scope['target_id']),", + " 'caseSensitiveScopesMatch': cache.scope(lower_scope['target_id']).generation == cache.scope(upper_scope['target_id']).generation,", + " 'caseSensitiveRootsMatch': cache.scope(lower_root['target_id']).generation == cache.scope(upper_root['target_id']).generation,", + " 'linkedWorktreeMatches': cache.scope(linked_scope['target_id']).generation == cache.scope(upper_scope['target_id']).generation,", + " 'sameOriginCloneMatches': cache.scope(clone_scope['target_id']).generation == cache.scope(upper_scope['target_id']).generation,", " 'exactResolvedPathMatches': history._same_repository(connection, same_path, upper_scope, identities=cache),", "}", - "connection.execute('INSERT INTO scans VALUES (?, ?, ?, ?, ?)', ('lower-scan', lower_scope['target_id'], lower_scope['target_path'], 'complete', '2026-08-15T00:00:00Z'))", + "connection.execute('INSERT INTO scans VALUES (?, ?, ?, ?, ?, ?)', ('lower-scan', lower_scope['target_id'], lower_scope['target_path'], lower_scope['repository_generation'], 'complete', '2026-08-15T00:00:00Z'))", "history._same_repository = lambda *args, **kwargs: False", "selected = history.list_unmatched_scan_pairs(connection, SimpleNamespace(repository=upper_scope['target_path'], force=False), backfill_finding_details=lambda *args: None, read_coverage=lambda scan: {})", "checks['caseSensitiveUnmatchedScanCount'] = selected['scanCount']", diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts index ddd5c49c4..1a7b338d0 100644 --- a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -569,6 +569,166 @@ with ExitStack() as stack: "SELECT repository_identity FROM security_targets WHERE id = 'changed'" ).fetchone()[0], })) + elif scenario == "selected-latest": + add_target("requested", "repository-current") + add_target("linked", "repository-current") + add_target("unrelated", "repository-other") + add_scan("legacy", "requested", generation=None) + add_finding("legacy", "legacy-only") + add_finding("legacy", "same-key", closed=True) + add_scan("bound", "linked") + add_finding("bound", "bound-only") + add_finding("bound", "same-key") + add_scan("other", "unrelated") + add_finding("other", "other-only") + selected = findings("requested") + unscoped = list(indexes._indexed_findings(connection)) + print(json.dumps({ + "selected": {row["occurrenceId"]: [row["confirmedInLatestScan"], row["status"], row["occurrenceCount"]] for row in selected}, + "unscoped": {row["occurrence_id"]: row["confirmed_in_latest_scan"] for row in unscoped}, + })) + elif scenario == "archive-recovery": + database = sqlite3.connect(":memory:") + database.row_factory = sqlite3.Row + database.execute("CREATE TABLE scans(id TEXT PRIMARY KEY, scan_dir TEXT UNIQUE)") + scan_dir = root / "outputs" / "scan" + archive = scan_dir.with_name(scan_dir.name + ".previous-synthetic") + + def recover(*, expected="previous", owner="previous", archived_owner=None, + original_exists=True, nonempty=False, invalid_sibling=False, + invalid_directory=False, rename_fails=False): + selected_archive = archive if not invalid_sibling else scan_dir.with_name("unrelated") + database.execute("DELETE FROM scans") + if owner is not None: + database.execute("INSERT INTO scans VALUES (?, ?)", (owner, str(scan_dir))) + if archived_owner is not None: + database.execute("INSERT INTO scans VALUES (?, ?)", (archived_owner, str(selected_archive))) + database.commit() + present = {selected_archive, *((scan_dir,) if original_exists else ())} + operations = [] + lock_checks = [] + + def canonical(path): + lock_checks.append(database.in_transaction) + if path not in present or invalid_directory: + raise SystemExit("Synthetic invalid scan directory") + return path + + def lstat(path): + if path not in present: + raise FileNotFoundError(errno.ENOENT, "Synthetic missing path", str(path)) + return SimpleNamespace(st_mode=stat.S_IFDIR) + + def rmdir(path): + lock_checks.append(database.in_transaction) + if nonempty: + raise OSError(errno.ENOTEMPTY, "Synthetic nonempty output") + present.remove(path) + operations.append("rmdir") + + def rename(path, destination): + lock_checks.append(database.in_transaction) + if rename_fails: + raise OSError("Synthetic rename failure") + assert destination not in present + present.remove(path) + present.add(destination) + operations.append("rename") + + with patch.object(Path, "resolve", lambda path, *a, **k: path), \ + patch.object(Path, "lstat", lstat), \ + patch.object(Path, "rmdir", rmdir), \ + patch.object(Path, "rename", rename): + try: + result = scan_start.restore_cli_scan_archive(database, argparse.Namespace( + scan_dir=str(scan_dir), archived_scan_dir=str(selected_archive), + previous_scan_id=expected, previous_scan_absent=expected is None, + ), canonical)["disposition"] + except (SystemExit, OSError): + result = "error" + return { + "disposition": result, "operations": operations, + "originalPresent": scan_dir in present, "archivePresent": selected_archive in present, + "locked": all(lock_checks), "transactionClosed": not database.in_transaction, + "owners": dict(database.execute("SELECT scan_dir, id FROM scans")), + } + + base = ["workbench", "restore-cli-scan-archive", "--scan-dir", str(scan_dir), "--archived-scan-dir", str(archive)] + parser_accepts = [] + for flags in (["--previous-scan-id", "previous"], ["--previous-scan-absent"], [], ["--previous-scan-id", "previous", "--previous-scan-absent"]): + with patch.object(sys, "argv", [*base, *flags]), redirect_stderr(io.StringIO()): + try: + workbench.parse_args("Synthetic archive recovery") + except SystemExit: + parser_accepts.append(False) + else: + parser_accepts.append(True) + cases = { + "recorded": recover(), + "unrecorded": recover(expected=None, owner=None), + "missingOriginal": recover(original_exists=False), + "committed": recover(owner="new", archived_owner="previous"), + "changedOwner": recover(owner="other"), + "unexpectedOwner": recover(expected=None), + "nonempty": recover(nonempty=True), + "invalidSibling": recover(invalid_sibling=True), + "invalidDirectory": recover(invalid_directory=True), + "renameFailure": recover(rename_fails=True), + } + print(json.dumps({"cases": cases, "parserAccepts": parser_accepts, "scanDir": str(scan_dir), "archiveDir": str(archive)})) + elif scenario == "archive-registration": + target = root / "requested" + scan_dir = root / "outputs" / "scan" + cases = {} + class ReadyToInsert(Exception): + pass + for case in ("valid", "owner-refused", "parent-refused", "nonempty", "noncanonical"): + events = [] + def canonical(path): + events.append(["canonical", connection.in_transaction]) + if case == "noncanonical" and connection.in_transaction: + raise SystemExit("Synthetic changed output") + return path + def entries(path): + assert path == scan_dir + events.append(["empty", connection.in_transaction]) + return iter(["retained-artifact"] if case == "nonempty" and connection.in_transaction else []) + def register(database, path): + events.append(["owner", database.in_transaction]) + if case == "owner-refused": + raise SystemExit("Synthetic refused owner") + return state.RegisteredRepositoryTarget("requested", "current") + def parent(database, scan_id): + events.append(["parent", database.in_transaction]) + return {"target_id": "requested", "repository_generation": "other" if case == "parent-refused" else "current"} + def archive_scan(database, *args): + events.append(["archive", database.in_transaction]) + raise ReadyToInsert() + with ExitStack() as mocks: + for name, value in { + "require_target": target, + "parse_scan_recipe": {"target": {"kind": "repository", "paths": []}, "mode": "standard"}, + "scan_target_identity": ("synthetic", None, 7, 8), + "directory_snapshot_regular_file_count": 0, + }.items(): + mocks.enter_context(patch.object(workbench, name, return_value=value)) + mocks.enter_context(patch.object(workbench, "require_scannable_target")) + mocks.enter_context(patch.object(workbench, "require_uuid", side_effect=lambda value, label: value)) + mocks.enter_context(patch.object(workbench, "require_canonical_scan_directory", side_effect=canonical)) + mocks.enter_context(patch.object(workbench, "register_security_target", side_effect=register)) + mocks.enter_context(patch.object(workbench, "require_scan", side_effect=parent)) + mocks.enter_context(patch.object(workbench, "archive_scan", side_effect=archive_scan)) + mocks.enter_context(patch.object(Path, "iterdir", entries)) + try: + workbench.register_cli_scan(connection, argparse.Namespace( + repository=str(target), scan_dir=str(scan_dir), recipe_json="{}", parent_scan_id="parent", + )) + except ReadyToInsert: + accepted = True + except SystemExit: + accepted = False + cases[case] = {"accepted": accepted, "events": events, "transactionClosed": not connection.in_transaction} + print(json.dumps(cases)) elif scenario == "completion-order": add_target("first", "repository-current") add_target("second", "repository-current") @@ -1266,6 +1426,106 @@ test("orders completed history by database visibility across legacy and current }); }); +test("confirms findings against the latest selected scan without merging legacy groups", () => { + expect(run("selected-latest")).toEqual({ + selected: { + "legacy:legacy-only": [false, "open", 1], + "legacy:same-key": [false, "closed", 1], + "bound:bound-only": [true, "open", 1], + "bound:same-key": [true, "open", 1], + }, + unscoped: { + "legacy:legacy-only": true, + "legacy:same-key": true, + "bound:bound-only": true, + "bound:same-key": true, + "other:other-only": true, + }, + }); +}); + +test("restores only an unclaimed empty scan output under the workbench writer lock", () => { + const result = run("archive-recovery"); + const cases = result["cases"] as Record>; + expect(result["parserAccepts"]).toEqual([true, true, false, false]); + for (const name of ["recorded", "unrecorded", "missingOriginal"]) { + expect(cases[name]).toMatchObject({ + disposition: "restored", + operations: name === "missingOriginal" ? ["rename"] : ["rmdir", "rename"], + originalPresent: true, + archivePresent: false, + }); + } + expect(cases["committed"]).toMatchObject({ + disposition: "already-recorded", + operations: [], + owners: { + [String(result["scanDir"])]: "new", + [String(result["archiveDir"])]: "previous", + }, + }); + for (const name of ["changedOwner", "unexpectedOwner"]) { + expect(cases[name]).toMatchObject({ + disposition: "ownership-changed", + operations: [], + originalPresent: true, + archivePresent: true, + }); + } + for (const name of ["nonempty", "invalidSibling", "invalidDirectory"]) { + expect(cases[name]).toMatchObject({ + disposition: "error", + operations: [], + originalPresent: true, + archivePresent: true, + }); + } + expect(cases["renameFailure"]).toMatchObject({ + disposition: "error", + operations: ["rmdir"], + archivePresent: true, + }); + for (const value of Object.values(cases)) { + expect(value).toMatchObject({ locked: true, transactionClosed: true }); + } +}); + +test("rechecks scan output inside registration after owner and parent validation", () => { + const result = run("archive-registration") as Record< + string, + { + accepted: boolean; + events: Array<[string, boolean]>; + transactionClosed: boolean; + } + >; + expect(result["valid"]).toEqual({ + accepted: true, + events: [ + ["canonical", false], + ["empty", false], + ["owner", true], + ["parent", true], + ["canonical", true], + ["empty", true], + ["archive", true], + ], + transactionClosed: true, + }); + for (const name of [ + "owner-refused", + "parent-refused", + "nonempty", + "noncanonical", + ]) { + expect(result[name]?.accepted).toBe(false); + expect(result[name]?.events.some(([event]) => event === "archive")).toBe( + false, + ); + expect(result[name]?.transactionClosed).toBe(true); + } +}); + test("keeps authenticated historical aliases visible without trusting a replacement checkout", () => { const result = run("persisted-alias"); diff --git a/sdk/typescript/tests-ts/workbench-scan-history.test.ts b/sdk/typescript/tests-ts/workbench-scan-history.test.ts index 46cd19880..cde760442 100644 --- a/sdk/typescript/tests-ts/workbench-scan-history.test.ts +++ b/sdk/typescript/tests-ts/workbench-scan-history.test.ts @@ -11,8 +11,10 @@ test("loads each scan's matching findings once across historical batches", () => const probe = [ "import argparse, json, sqlite3, sys", + "from pathlib import Path", "sys.path.insert(0, sys.argv[1])", "import workbench_scan_history as history", + "repository = str(Path(sys.argv[2]).resolve())", "connection = sqlite3.connect(':memory:')", "connection.row_factory = sqlite3.Row", "connection.executescript('''", @@ -23,17 +25,18 @@ test("loads each scan's matching findings once across historical batches", () => "CREATE TABLE finding_triage (occurrence_id TEXT, status TEXT, close_reason TEXT);", "CREATE TABLE finding_locations (occurrence_id TEXT, relative_path TEXT, role TEXT, sort_order INTEGER);", "''')", + "connection.execute('INSERT INTO security_targets VALUES (?, ?)', ('target', repository))", "for index in range(3):", " scan = f'scan-{index}'", - " connection.execute('INSERT INTO scans VALUES (?, ?, NULL, ?, ?, ?)', (scan, sys.argv[2], 'complete', f'2026-08-01T0{index}:00:00Z', f'2026-08-01T0{index + 3}:00:00Z'))", + " connection.execute('INSERT INTO scans VALUES (?, ?, ?, ?, ?, ?)', (scan, repository, 'target', 'complete', f'2026-08-01T0{index}:00:00Z', f'2026-08-01T0{index + 3}:00:00Z'))", " connection.execute('INSERT INTO finding_occurrences VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (scan, scan, scan, '{}', 'fix', 'high', 'summary', 'title'))", "queries = []", "connection.set_trace_callback(queries.append)", "backfilled = []", - "result = history.list_unmatched_scan_pairs(connection, argparse.Namespace(repository=sys.argv[2], force=False), backfill_finding_details=lambda _connection, scan: backfilled.append(scan['id']), read_coverage=lambda _scan: {})", + "result = history.list_unmatched_scan_pairs(connection, argparse.Namespace(repository=repository, force=False), backfill_finding_details=lambda _connection, scan: backfilled.append(scan['id']), read_coverage=lambda _scan: {})", "finding_queries = sum('FROM finding_occurrences AS occurrences' in query for query in queries)", "def planned(focus=None):", - " value = history.list_unmatched_scan_pairs(connection, argparse.Namespace(repository=sys.argv[2], force=False, after_scan_id=focus), backfill_finding_details=lambda *_: None, read_coverage=lambda _: {})", + " value = history.list_unmatched_scan_pairs(connection, argparse.Namespace(repository=repository, force=False, after_scan_id=focus), backfill_finding_details=lambda *_: None, read_coverage=lambda _: {})", " return {'batches': [{'after': batch['afterScanId'], 'before': [scan['scanId'] for scan in batch['beforeScans']]} for batch in value['batches']], 'skipped': value['skippedPairs']}", "connection.execute(\"UPDATE scans SET status = 'running', completed_at = NULL WHERE id = 'scan-0'\")", "later_completed_first = planned('scan-2')", @@ -65,7 +68,7 @@ test("loads each scan's matching findings once across historical batches", () => { encoding: "utf8", timeout: 10_000 }, ); - expect(result.status).toBe(0); + expect(result.status, result.stderr).toBe(0); expect(result.stderr).toBe(""); expect(JSON.parse(result.stdout)).toMatchObject({ backfilled: ["scan-0", "scan-1", "scan-2"], From 4bbb75e98c032b3089b9a9676c61ab4b5a74742c Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sun, 16 Aug 2026 09:23:23 -0700 Subject: [PATCH 16/21] Bound repository identity maintenance to selected targets --- sdk/typescript/README.md | 6 +- .../scripts/workbench_schema.py | 4 - .../scripts/workbench_target_state.py | 37 +++++++--- .../tests-ts/workbench-identity-cache.test.ts | 73 ++++++++++++++++++- 4 files changed, 103 insertions(+), 17 deletions(-) diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index aa594c501..b61a288f5 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -671,7 +671,11 @@ generation in the selected state database. The generation identifies the shared Git and primary object-store directory instances; it is not a repository UUID. History written without that scan-level evidence stays attached to its original target and scan IDs. Older clients' history is not retroactively shared across -worktrees. Explicit comparisons between independently verified copies of the same +worktrees. Ordinary database opens do not retry unrelated unscanned targets; +eligible targets are bound when explicitly registered. On older Git, primary +`.git` files also need a repository-side `core.worktree` record identifying the +checkout. Layouts without that reverse record retain target-local history. +Explicit comparisons between independently verified copies of the same repository remain available without joining their finding histories. Findings indicate whether they were confirmed in the repository's latest completed scan across those linked worktrees. Finding confirmation and diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py index 719daebe4..c92162d2c 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py @@ -813,10 +813,6 @@ def apply_migrations( repair_deep_scan_failure_counter_migration(connection) if should_backfill_targets: backfill_security_targets(connection) - elif any(version == 31 for version, _, _ in migrations): - from workbench_target_state import backfill_repository_identities - - backfill_repository_identities(connection) connection.commit() except BaseException: connection.rollback() diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index cd3f11f15..093c9d425 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -48,8 +48,10 @@ def _repository_worktree(target: Path) -> tuple[Path, str] | None: return canonical_root, relative.as_posix() -def _path_from_git_bytes(value: bytes, relative_to: Path) -> Path | None: - if value.endswith(b"\n"): +def _path_from_git_bytes( + value: bytes, relative_to: Path, *, strip_line_feed: bool = True +) -> Path | None: + if strip_line_feed and value.endswith(b"\n"): value = value[:-1] if not value or b"\0" in value: return None @@ -70,24 +72,39 @@ def _registered_worktree(target: Path, root: Path, common: Path) -> bool: for record in registered.split(b"\0") if record.startswith(b"worktree ") ) - # Git before 2.36 emits unquoted newline-delimited paths. Its reciprocal - # administrative files are unambiguous even when a checkout path has newlines. + # Git before 2.36 emits unquoted newline-delimited paths. Repository-side + # ownership records are unambiguous even when a checkout path has newlines. gitdir = _git_path(target, "rev-parse", "--absolute-git-dir") dotgit = root / ".git" try: mode = dotgit.lstat().st_mode if stat.S_ISDIR(mode): return gitdir == common == Path(os.path.realpath(dotgit)) - if ( - not stat.S_ISREG(mode) or gitdir is None - or gitdir.parent != common / "worktrees" or not gitdir.is_dir() - ): + if not stat.S_ISREG(mode) or gitdir is None or not gitdir.is_dir(): return False forward = dotgit.read_bytes() - if not forward.startswith(b"gitdir: "): + if ( + not forward.startswith(b"gitdir: ") + or _path_from_git_bytes(forward[len(b"gitdir: "):], root) != gitdir + ): return False + if gitdir == common: + configured = git_bytes( + target, "config", "--null", "--show-scope", "--no-includes", + "--get", "core.worktree", + ) + fields = configured.split(b"\0") if configured is not None else [] + if ( + len(fields) != 3 or fields[0] not in (b"local", b"worktree") + or not fields[1] or fields[2] + ): + return False + configured_root = _path_from_git_bytes( + fields[1], gitdir, strip_line_feed=False + ) + return configured_root is not None and str(configured_root) == str(root) return ( - _path_from_git_bytes(forward[len(b"gitdir: "):], root) == gitdir + gitdir.parent == common / "worktrees" and _path_from_git_bytes((gitdir / "gitdir").read_bytes(), gitdir) == Path(os.path.realpath(dotgit)) ) diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts index 1a7b338d0..1035eca8d 100644 --- a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -229,7 +229,8 @@ with ExitStack() as stack: common = main / ".git" admin = common / "worktrees" / "linked" objects = common / "objects" - active = {"root": main, "gitdir": common, "nul": False} + active = {"root": main, "gitdir": common, "nul": False, "config": None} + config_command = ("config", "--null", "--show-scope", "--no-includes", "--get", "core.worktree") commands = [] directory = lambda inode, birth: SimpleNamespace( st_mode=stat.S_IFDIR, st_dev=7, st_ino=inode, st_birthtime_ns=birth @@ -247,6 +248,8 @@ with ExitStack() as stack: commands.append(arguments) if arguments == ("worktree", "list", "--porcelain", "-z"): return b"worktree " + os.fsencode(active["root"]) + b"\0\0" if active["nul"] else None + if arguments == config_command: + return active["config"] paths_by_command = { ("rev-parse", "--show-toplevel"): active["root"], ("rev-parse", "--path-format=absolute", "--git-common-dir"): common, @@ -255,6 +258,14 @@ with ExitStack() as stack: } value = paths_by_command.get(arguments) return os.fsencode(value) + b"\n" if value is not None else None + primary = root / "primary-checkout" + def file_primary(configuration, selected_root=primary, forward=common): + active.update(root=selected_root, gitdir=common, nul=False, config=configuration) + records[str(selected_root / ".git")] = SimpleNamespace(st_mode=stat.S_IFREG) + files[str(selected_root / ".git")] = b"gitdir: " + os.fsencode(forward) + b"\n" + return real_identity_details(selected_root) is not None + def configured(value, scope=b"local"): + return scope + b"\0" + os.fsencode(value) + b"\0" with patch.object(state, "git_bytes", git_bytes), \ patch.object(os.path, "realpath", side_effect=os.path.abspath), \ patch.object(Path, "stat", lambda path, *a, **k: records[str(path)]), \ @@ -272,6 +283,26 @@ with ExitStack() as stack: nul_linked = real_identity_details(linked) records[str(objects)] = directory(23, 400) changed = real_identity_details(linked) + literal_tilde = common / "~literal" / "checkout" + trailing_lf = root / "primary-checkout\n" + trailing_space = root / "primary-checkout " + primary_results = { + "localAbsolute": file_primary(configured(primary)), + "worktreeRelative": file_primary(configured(os.path.relpath(primary, common), b"worktree")), + "literalLeadingTilde": file_primary(configured(Path("~literal") / "checkout"), literal_tilde), + "trailingLineFeed": file_primary(configured(trailing_lf), trailing_lf), + "trailingWhitespace": file_primary(configured(trailing_space), trailing_space), + "missing": file_primary(None), + "wrongScopes": all(not file_primary(configured(primary, scope)) for scope in ( + b"global", b"system", b"command", b"unknown", + )), + "malformed": all(not file_primary(value) for value in ( + b"local\0", b"local\0\0", configured(primary)[:-1], + configured(primary) + b"extra\0", + )), + "wrongRoot": file_primary(configured(main)), + "foreignForward": file_primary(configured(primary), forward=root / "other-common"), + } print(json.dumps({ "oldMain": first is not None, "oldLinkedMatches": old_linked.value == first.value, @@ -282,6 +313,9 @@ with ExitStack() as stack: "legacyMaterialUnchanged": changed.legacy_value == first.legacy_value, "domainSeparated": first.value != first.legacy_value, "actualObjectLookup": ("rev-parse", "--path-format=absolute", "--git-path", "objects") in commands, + "oldPrimary": primary_results, + "rawConfigLookup": any(command == config_command for command in commands) + and all(command == config_command for command in commands if command[0] == "config"), })) elif scenario == "scan-generation": for name, value in (("requested", "generation-current"), ("alias", "generation-current"), @@ -1041,6 +1075,8 @@ with ExitStack() as stack: add_target("requested", "repository-current") add_scan("requested-scan", "requested") add_target("unscanned", None, "repository-current") + add_target("unselected", None, "repository-current") + details.pop(paths["unselected"]) add_target("historical", None, "repository-current") add_scan("historical-scan", "historical") add_finding("historical-scan", "historical-finding", closed=True) @@ -1049,6 +1085,13 @@ with ExitStack() as stack: apply_migrations(connection, MIGRATIONS, lambda: timestamp, backfill) apply_migrations(connection, MIGRATIONS, lambda: timestamp, backfill) maintenance_probes = dict(probes) + probes.clear() + with patch.object(state, "_bind_unscanned_repository_identity", wraps=state._bind_unscanned_repository_identity) as guarded_bind: + selected_id = state.ensure_security_target(connection, paths["unscanned"]) + selected_guard = guarded_bind.call_count == 1 and guarded_bind.call_args.args[1:] == ( + "unscanned", paths["unscanned"], "repository-current", + ) + selected_probes = dict(probes) historical_id = state.ensure_security_target(connection, paths["historical"]) add_scan("established-scan", "unscanned") missing.update((paths["unscanned"], paths["historical"])) @@ -1058,6 +1101,9 @@ with ExitStack() as stack: )}, "fullBackfillCalls": backfill.call_count, "maintenanceProbes": {name: maintenance_probes.get(path, 0) for name, path in paths.items()}, + "selectedProbes": {name: selected_probes.get(path, 0) for name, path in paths.items()}, + "selectedRegistrationId": selected_id, + "selectedGuarded": selected_guard, "historicalRegistrationId": historical_id, "aliases": sorted(indexes.repository_target_ids(connection, "requested")), "scans": listed("requested"), @@ -1274,6 +1320,19 @@ test("uses registered old-Git paths and stable object-directory generation evide legacyMaterialUnchanged: true, domainSeparated: true, actualObjectLookup: true, + oldPrimary: { + localAbsolute: true, + worktreeRelative: true, + literalLeadingTilde: true, + trailingLineFeed: true, + trailingWhitespace: true, + missing: false, + wrongScopes: true, + malformed: true, + wrongRoot: false, + foreignForward: false, + }, + rawConfigLookup: true, }); }); @@ -1586,20 +1645,30 @@ test("keeps unproved legacy history unbound and rejects newer or indeterminate o expect(result["aliases"]).toEqual(["replacement-alias"]); }); -test("revisits only eligible late NULL identities on ordinary database opens", () => { +test("binds late NULL identities only when the selected target is registered", () => { const result = run("late-null"); expect(result["stored"]).toEqual({ requested: "repository-current", unscanned: "repository-current", + unselected: null, historical: null, }); expect(result["fullBackfillCalls"]).toBe(0); expect(result["maintenanceProbes"]).toEqual({ + requested: 0, + unscanned: 0, + unselected: 0, + historical: 0, + }); + expect(result["selectedProbes"]).toEqual({ requested: 0, unscanned: 1, + unselected: 0, historical: 0, }); + expect(result["selectedRegistrationId"]).toBe("unscanned"); + expect(result["selectedGuarded"]).toBe(true); expect(result["historicalRegistrationId"]).toBe("historical"); expect(result["aliases"]).toEqual(["requested", "unscanned"]); expect(result["scans"]).toEqual(["established-scan", "requested-scan"]); From 10a6dd767444591809cddc1f812ad817274e4037 Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Sun, 16 Aug 2026 12:23:00 -0700 Subject: [PATCH 17/21] Repair legacy identity upgrades and precompute repository counts --- .../scripts/workbench_native_indexes.py | 58 +++-- .../scripts/workbench_schema.py | 17 +- .../tests-ts/workbench-identity-cache.test.ts | 210 ++++++++++++++++-- .../workbench-target-migration.test.ts | 40 +++- 4 files changed, 274 insertions(+), 51 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py index 693eccf2f..968dd64b7 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_native_indexes.py @@ -3,6 +3,7 @@ import argparse import sqlite3 import sys +from collections import Counter from collections.abc import Iterator from itertools import islice from pathlib import Path @@ -317,17 +318,34 @@ def list_repositories( latest_scan_by_target.setdefault(row["target_id"], scans_by_id[row["id"]]) targets = {row["id"]: row for row in connection.execute("SELECT * FROM security_targets")} - - scan_evidence = { - row["id"]: row for row in connection.execute("SELECT * FROM scans") + query = args.query.strip().casefold() if args is not None and args.query else "" + selected_targets = [ + (target_id, latest_scan, target) + for target_id, latest_scan in latest_scan_by_target.items() + if (target := targets.get(target_id)) is not None + and (args is None or args.target_id is None or target_id == args.target_id) + and (args is None or args.status != "not_scanned") + and ( + not query + or query in target["display_name"].casefold() + or query in target["current_path"].casefold() + ) + ] + scopes = { + target_id: identities.scope(target_id) + for target_id, _, _ in selected_targets } - open_findings = [ - row for row in _indexed_findings(connection, identities=identities) + open_findings_by_group = Counter( + scan_repository_group(row) + for row in _indexed_findings(connection, identities=identities) if row["status"] == "open" - ] + ) if any( + scope.available and not scope.exact_target for scope in scopes.values() + ) else Counter() - def open_findings_count(target_id: str) -> int: - scope = identities.scope(target_id) + def open_findings_count(scope: RepositoryScanScope) -> int: + if not scope.available: + return 0 if scope.exact_target: return sum( row["status"] == "open" @@ -335,38 +353,32 @@ def open_findings_count(target_id: str) -> int: connection, identities=identities, scan_scope=scope ) ) - return sum( - any(scope.contains(scan_evidence[scan_id]) for scan_id in row["known_scan_ids"]) - for row in open_findings + return ( + open_findings_by_group[("repository", scope.generation)] + if identities.supports_generation and scope.generation is not None else 0 + ) + ( + open_findings_by_group[("target", scope.target_id)] if scope.target_id else 0 ) + repositories = [ { "checkoutAvailable": Path(target["current_path"]).is_dir(), "displayName": target["display_name"], "latestScan": latest_scan, - "openFindingsCount": open_findings_count(target_id), + "openFindingsCount": open_findings_count(scopes[target_id]), "scanCount": scan_count_by_target[target_id], "targetId": target_id, "targetPath": target["current_path"], } - for target_id, latest_scan in latest_scan_by_target.items() - if (target := targets.get(target_id)) is not None + for target_id, latest_scan, target in selected_targets ] if args is None: return {"repositories": repositories} - query = args.query.strip().casefold() if args.query else "" repositories = [ repository for repository in repositories - if (args.target_id is None or repository["targetId"] == args.target_id) - and args.status != "not_scanned" - and (args.status != "open_findings" or repository["openFindingsCount"] > 0) - and ( - not query - or query in repository["displayName"].casefold() - or query in repository["targetPath"].casefold() - ) + if args.status != "open_findings" or repository["openFindingsCount"] > 0 ] if args.limit is None and args.offset == 0: return {"repositories": repositories} diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py index c92162d2c..21711ad3a 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py @@ -943,9 +943,6 @@ def normalize_pre_release_migrations(connection: sqlite3.Connection, timestamp: repository_identity_migration is not None and repository_identity_migration["name"] == "persist repository identities" ): - from workbench_target_state import normalize_pre_release_repository_identities - - normalize_pre_release_repository_identities(connection) connection.execute( "UPDATE schema_migrations SET version = 31 WHERE version = 30 AND name = ?", ("persist repository identities",), @@ -1335,6 +1332,20 @@ def repair_repository_identity_migration(connection: sqlite3.Connection) -> bool "AND name = 'security_targets_by_repository_identity'" ).fetchone() identity_changed = "repository_identity" not in columns or index_exists is None + if ( + "repository_identity" in columns + and connection.execute( + "SELECT 1 FROM schema_migrations WHERE version = 31 AND name = ?", + ("persist repository identities",), + ).fetchone() is not None + and not any( + row["name"] == "repository_generation" + for row in connection.execute("PRAGMA table_info(scans)") + ) + ): + from workbench_target_state import normalize_pre_release_repository_identities + + normalize_pre_release_repository_identities(connection) migration_sql = next(sql for version, _, sql in MIGRATIONS if version == 31) for statement in sql_statements(migration_sql): if statement.startswith("ALTER TABLE security_targets"): diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts index 1035eca8d..9f2cf26fb 100644 --- a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -42,13 +42,20 @@ legacy_identity_migration = ( "ALTER TABLE security_targets ADD COLUMN repository_identity TEXT;\n" "CREATE INDEX security_targets_by_repository_identity ON security_targets(repository_identity);\n", ) +recorded_identity31 = scenario == "migration-recorded31" initial = ( (*tuple(item for item in MIGRATIONS if item[0] <= 28), legacy_identity_migration) if scenario in ("migration", "v30-current", "completion-order") else tuple(item for item in MIGRATIONS if item[0] <= 30) - if scenario == "null-history" else MIGRATIONS + if scenario == "null-history" or recorded_identity31 else MIGRATIONS ) apply_migrations(connection, initial, lambda: timestamp, lambda database: None) +if recorded_identity31: + connection.executescript(legacy_identity_migration[2]) + connection.execute( + "INSERT INTO schema_migrations VALUES (?, ?, ?)", + (31, legacy_identity_migration[1], timestamp), + ) paths = {} details = {} metadata = {} @@ -368,6 +375,69 @@ with ExitStack() as stack: "weakBindingPreserved": connection.execute("SELECT repository_identity FROM security_targets WHERE id = 'weak'").fetchone()[0] == legacy.legacy_value, "absentExactCounts": [absent_findings, absent_repository["openFindingsCount"]], })) + elif scenario == "repository-counts": + for name, stored, live in ( + ("first", "shared-generation", None), + ("second", "shared-generation", None), + ("absent", "shared-generation", None), + ("independent", "other-generation", None), + ("refused", "previous-generation", "current-generation"), + ): + add_target(name, stored, live) + add_scan(name + "-scan", name) + add_scan("first-legacy", "first", generation=None) + add_scan("second-legacy", "second", generation=None) + add_finding("first-scan", "shared-open") + add_finding("second-scan", "shared-open") + add_finding("first-legacy", "first-local") + add_finding("second-legacy", "second-local") + reviewed = add_finding("first-scan", "reviewed-original", closed=True) + renamed = add_finding("absent-scan", "reviewed-renamed") + add_finding("independent-scan", "independent-open") + add_finding("refused-scan", "refused-open") + connection.execute("INSERT INTO scan_comparisons VALUES (?, ?, ?, ?, ?)", + ("first-scan", "absent-scan", "{}", timestamp, timestamp)) + connection.execute("INSERT INTO scan_comparison_matches VALUES (?, ?, ?, ?, ?)", + ("first-scan", "absent-scan", reviewed, renamed, "Synthetic match")) + missing.add(paths["absent"]) + real_index = indexes._indexed_findings + real_inspect = state._inspect_repository_target + def repository_counts(**options): + args = argparse.Namespace(query=None, target_id=None, status=None, limit=None, offset=0) + vars(args).update(options) + calls, inspections = [], [] + def indexed(database, **kwargs): + scope = kwargs.get("scan_scope") + calls.append(None if scope is None else scope.target_id) + return real_index(database, **kwargs) + def inspected(database, target_id, *args, **kwargs): + inspections.append(target_id) + return real_inspect(database, target_id, *args, **kwargs) + with patch.object(indexes, "_indexed_findings", side_effect=indexed), \ + patch.object(state, "_inspect_repository_target", side_effect=inspected): + result = indexes.list_repositories(connection, args) + return { + "counts": {row["targetId"]: row["openFindingsCount"] for row in result["repositories"]}, + "ids": [row["targetId"] for row in result["repositories"]], + "calls": calls, "inspections": sorted(inspections), + "nextOffset": result.get("nextOffset"), + } + all_repositories = repository_counts() + direct_counts = { + name: sum(row["status"] == "open" for row in findings(name)) for name in paths + } + open_repositories = repository_counts(status="open_findings") + page = repository_counts(status="open_findings", limit=2, offset=1) + print(json.dumps({ + "all": all_repositories, + "directCounts": direct_counts, + "absentOnly": repository_counts(target_id="absent"), + "queryOnly": repository_counts(query="INDEPENDENT"), + "refusedOnly": repository_counts(target_id="refused"), + "notScanned": repository_counts(status="not_scanned"), + "openIds": open_repositories["ids"], + "pagePreserved": page["ids"] == open_repositories["ids"][1:3] and page["nextOffset"] == 3, + })) elif scenario == "scan-writers": add_target("writer", "generation-writer") target = Path(paths["writer"]) @@ -1277,12 +1347,40 @@ with ExitStack() as stack: "old-basic", "old-description", "no-scans", "scope-upper", "scope-lower" } else None ) - apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) - first = { - row["id"]: row["repository_identity"] - for row in connection.execute("SELECT id, repository_identity FROM security_targets") + add_finding("old-basic-scan", "historical-finding", closed=True) + connection.execute( + "INSERT INTO scan_artifacts VALUES (?, ?, ?, ?)", + ("old-basic-scan", "findings", str(root / "historical-findings.json"), timestamp), + ) + tables = ("scans", "findings", "finding_occurrences", "finding_triage", "scan_artifacts") + columns = { + table: ", ".join(row["name"] for row in connection.execute("PRAGMA table_info(" + table + ")")) + for table in tables } - apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) + def retained_records(): + return {table: [tuple(row) for row in connection.execute( + "SELECT " + columns[table] + " FROM " + table + )] for table in tables} + retained = retained_records() + starting_version = connection.execute( + "SELECT version FROM schema_migrations WHERE name = ?", + ("persist repository identities",), + ).fetchone()[0] + real_normalize = state.normalize_pre_release_repository_identities + normalization_transactions = [] + def normalize(database): + normalization_transactions.append(database.in_transaction) + return real_normalize(database) + with patch.object(state, "normalize_pre_release_repository_identities", side_effect=normalize) as normalizer: + apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) + first = { + row["id"]: row["repository_identity"] + for row in connection.execute("SELECT id, repository_identity FROM security_targets") + } + probes.clear() + apply_migrations(connection, MIGRATIONS, lambda: timestamp, state.backfill_security_targets) + current_open_probes = sum(probes.values()) + normalization_count = normalizer.call_count second = { row["id"]: row["repository_identity"] for row in connection.execute("SELECT id, repository_identity FROM security_targets") @@ -1290,6 +1388,12 @@ with ExitStack() as stack: print(json.dumps({ "identities": first, "expected": expected, + "startingVersion": starting_version, + "normalizationCount": normalization_count, + "normalizationTransactions": normalization_transactions, + "currentOpenProbes": current_open_probes, + "recordsPreserved": retained == retained_records(), + "scanGenerationsUnbound": all(row[0] is None for row in connection.execute("SELECT repository_generation FROM scans")), "idempotent": first == second, "foldedLegacyScopesEqual": originals["scope-upper"] == originals["scope-lower"], "currentScopesDistinct": first["scope-upper"] != first["scope-lower"], @@ -1360,6 +1464,67 @@ test("keeps automatic history bounded by saved scan generation", () => { expect(result["absentExactCounts"]).toEqual([1, 1]); }); +test("counts repository groups once while preserving absent exact-target decisions", () => { + const result = run("repository-counts"); + const all = result["all"] as { + counts: Record; + calls: Array; + inspections: string[]; + }; + + expect(all.counts).toEqual({ + first: 2, + second: 2, + absent: 1, + independent: 1, + refused: 0, + }); + expect(result["directCounts"]).toEqual(all.counts); + expect(all.calls).toEqual([null, "absent"]); + expect(all.inspections).toEqual([ + "absent", + "first", + "independent", + "refused", + "second", + ]); + expect(result["absentOnly"]).toEqual({ + counts: { absent: 1 }, + ids: ["absent"], + calls: ["absent"], + inspections: ["absent"], + nextOffset: null, + }); + expect(result["queryOnly"]).toEqual({ + counts: { independent: 1 }, + ids: ["independent"], + calls: [null], + inspections: ["independent"], + nextOffset: null, + }); + expect(result["refusedOnly"]).toEqual({ + counts: { refused: 0 }, + ids: ["refused"], + calls: [], + inspections: ["refused"], + nextOffset: null, + }); + expect(result["notScanned"]).toEqual({ + counts: {}, + ids: [], + calls: [], + inspections: [], + nextOffset: null, + }); + expect((result["openIds"] as string[]).sort()).toEqual([ + "absent", + "first", + "independent", + "second", + ]); + expect(result["pagePreserved"]).toBe(true); +}); + test("records generation explicitly in both transactional parent scan writers", () => { const result = run("scan-writers"); expect(result["generations"]).toEqual({ @@ -1617,18 +1782,29 @@ test("keeps authenticated historical aliases visible without trusting a replacem expect(result["stored"]).toBe("repository-current"); }); -test("upgrades only independently verified pre-release repository hashes", () => { - const result = run("migration"); +test.each(["migration", "migration-recorded31"])( + "upgrades only independently verified pre-release repository hashes (%s)", + (scenario) => { + const result = run(scenario); - expect(result["identities"]).toEqual(result["expected"]); - expect(result["idempotent"]).toBe(true); - expect(result["foldedLegacyScopesEqual"]).toBe(true); - expect(result["currentScopesDistinct"]).toBe(true); - expect(result["targetCount"]).toBe(12); - expect(result["migrations"]).toEqual( - Array.from({ length: 31 }, (_, index) => index + 1), - ); -}); + expect(result["identities"]).toEqual(result["expected"]); + expect(result["startingVersion"]).toBe( + scenario === "migration-recorded31" ? 31 : 30, + ); + expect(result["normalizationCount"]).toBe(1); + expect(result["normalizationTransactions"]).toEqual([true]); + expect(result["currentOpenProbes"]).toBe(0); + expect(result["recordsPreserved"]).toBe(true); + expect(result["scanGenerationsUnbound"]).toBe(true); + expect(result["idempotent"]).toBe(true); + expect(result["foldedLegacyScopesEqual"]).toBe(true); + expect(result["currentScopesDistinct"]).toBe(true); + expect(result["targetCount"]).toBe(12); + expect(result["migrations"]).toEqual( + Array.from({ length: 31 }, (_, index) => index + 1), + ); + }, +); test("keeps unproved legacy history unbound and rejects newer or indeterminate owners", () => { const result = run("null-history"); diff --git a/sdk/typescript/tests-ts/workbench-target-migration.test.ts b/sdk/typescript/tests-ts/workbench-target-migration.test.ts index ab9139154..c66e7b47d 100644 --- a/sdk/typescript/tests-ts/workbench-target-migration.test.ts +++ b/sdk/typescript/tests-ts/workbench-target-migration.test.ts @@ -195,13 +195,20 @@ if scenario == "out-of-order-publication-migrations": apply_migrations( connection, (*historical, identity_migration), lambda: timestamp, backfill ) -elif scenario == "pre-release-identity-version": - apply_migrations( - connection, - (*historical, legacy_identity_migration), - lambda: timestamp, - backfill, - ) +elif scenario in ("pre-release-identity-version", "pre-release-identity-version31"): + if scenario == "pre-release-identity-version": + apply_migrations( + connection, + (*historical, legacy_identity_migration), + lambda: timestamp, + backfill, + ) + else: + connection.executescript(legacy_identity_migration[2]) + connection.execute( + "INSERT INTO schema_migrations VALUES (?, ?, ?)", + (identity_version, legacy_identity_migration[1], timestamp), + ) connection.execute( "UPDATE security_targets SET repository_identity = 'synthetic-identity'" ) @@ -244,6 +251,11 @@ print(json.dumps({ "repositoryIdentityIndexIsUnique": bool( indexes["security_targets_by_repository_identity"]["unique"] ), + "hasRepositoryGenerationColumn": "repository_generation" in scan_columns, + "hasRepositoryGenerationIndex": "scans_by_repository_generation" in scan_indexes, + "repositoryGenerationColumnIsNullable": not bool( + scan_columns["repository_generation"]["notnull"] + ), "completionSequenceIsNullable": not bool( scan_columns["completion_sequence"]["notnull"] ), @@ -372,6 +384,10 @@ describe("stable workbench target migration", () => { "pre-release-identity-version", "quarantines an unverifiable pre-release identity without changing its target", ], + [ + "pre-release-identity-version31", + "repairs an already-renumbered pre-release identity migration", + ], ] as const)("%s: %s", (scenario) => { const python = Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py"); @@ -402,6 +418,9 @@ describe("stable workbench target migration", () => { publicationMigrations: Record; repositoryIdentityColumnIsNullable: boolean; repositoryIdentityIndexIsUnique: boolean; + hasRepositoryGenerationColumn: boolean; + hasRepositoryGenerationIndex: boolean; + repositoryGenerationColumnIsNullable: boolean; completionSequenceIsNullable: boolean; completionSequenceIndexIsUnique: boolean; completionSequenceTriggers: string[]; @@ -410,7 +429,9 @@ describe("stable workbench target migration", () => { teamOnlyPublicationIndexes: string[]; }; expect(result).toEqual({ - backfillCalls: scenario === "pre-release-identity-version" ? 0 : 1, + backfillCalls: scenario.startsWith("pre-release-identity-version") + ? 0 + : 1, hasRepositoryIdentityColumn: true, hasRepositoryIdentityIndex: true, identityVersion: 31, @@ -421,6 +442,9 @@ describe("stable workbench target migration", () => { }, repositoryIdentityColumnIsNullable: true, repositoryIdentityIndexIsUnique: false, + hasRepositoryGenerationColumn: true, + hasRepositoryGenerationIndex: true, + repositoryGenerationColumnIsNullable: true, completionSequenceIsNullable: true, completionSequenceIndexIsUnique: true, completionSequenceTriggers: [ From c829c5fbd2a2a0235b539f2494a6b91d8eb45bcf Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:18:09 -0700 Subject: [PATCH 18/21] test(workbench): isolate directory metadata on Python 3.14 --- sdk/typescript/tests-ts/workbench-identity-cache.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts index 9f2cf26fb..93c95c878 100644 --- a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -273,10 +273,12 @@ with ExitStack() as stack: return real_identity_details(selected_root) is not None def configured(value, scope=b"local"): return scope + b"\0" + os.fsencode(value) + b"\0" + # Keep directory checks inside the metadata fixture on every supported Python. with patch.object(state, "git_bytes", git_bytes), \ patch.object(os.path, "realpath", side_effect=os.path.abspath), \ patch.object(Path, "stat", lambda path, *a, **k: records[str(path)]), \ patch.object(Path, "lstat", lambda path, *a, **k: records[str(path)]), \ + patch.object(Path, "is_dir", lambda path, *a, **k: stat.S_ISDIR(records[str(path)].st_mode)), \ patch.object(Path, "read_bytes", lambda path: files[str(path)]), \ patch.object(state, "_repository_birth_time_ns", side_effect=lambda path, value: value.st_birthtime_ns): first = real_identity_details(main) From 27a1ad66245ce6764b321cc917c1e62d47ea64b1 Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:22:59 -0700 Subject: [PATCH 19/21] fix(sdk): exclude internal helpers from public declarations --- sdk/typescript/src/api.ts | 1 + sdk/typescript/tsconfig.build.json | 1 + 2 files changed, 2 insertions(+) diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts index b61727eb4..3f0c68928 100644 --- a/sdk/typescript/src/api.ts +++ b/sdk/typescript/src/api.ts @@ -2328,6 +2328,7 @@ interface ScanEventRunOptions { onObserverError?: (observer: ScanObserverName, error: unknown) => void; } +/** @internal */ export async function runScanEvents( options: ScanEventRunOptions, ): Promise { diff --git a/sdk/typescript/tsconfig.build.json b/sdk/typescript/tsconfig.build.json index ccf7d2ecb..cf223fc5e 100644 --- a/sdk/typescript/tsconfig.build.json +++ b/sdk/typescript/tsconfig.build.json @@ -7,6 +7,7 @@ "outDir": "dist", "declaration": true, "declarationMap": true, + "stripInternal": true, "sourceMap": true, "inlineSources": true, "noEmit": false, From 2d9965ea6bfbfa16ca5f893caf993805a8302979 Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Tue, 18 Aug 2026 07:17:15 -0700 Subject: [PATCH 20/21] test(workbench): send identity fixture through stdin --- sdk/typescript/tests-ts/workbench-identity-cache.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts index 93c95c878..6f128c26f 100644 --- a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -1406,12 +1406,15 @@ with ExitStack() as stack: function run(scenario: string): Record { const python = (Bun.which("python3") ?? Bun.which("python"))!; + // Keep the Python fixture off the Windows command line. const execution = spawnSync( python, - ["-I", "-B", "-c", probe, join(PLUGIN_ROOT, "scripts"), scenario], - { encoding: "utf8", timeout: 10_000 }, + ["-I", "-B", "-", join(PLUGIN_ROOT, "scripts"), scenario], + { input: probe, encoding: "utf8", timeout: 10_000 }, + ); + expect(execution.status, execution.error?.message ?? execution.stderr).toBe( + 0, ); - expect(execution.status, execution.stderr).toBe(0); return JSON.parse(execution.stdout) as Record; } From a2f5676dcef1c70a855ce672b0016cb4987bcd07 Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:16:26 -0700 Subject: [PATCH 21/21] fix(workbench): verify ownership before rejoining scans --- .../_bundled_plugin/.codex-plugin/plugin.json | 2 +- .../scripts/deep_scan_workbench.py | 6 +- .../_bundled_plugin/scripts/workbench_db.py | 4 + .../scripts/workbench_target_state.py | 12 ++ sdk/typescript/src/version.ts | 2 +- .../tests-ts/deep-scan-workbench.test.ts | 1 + .../tests-ts/workbench-identity-cache.test.ts | 133 ++++++++++++++++++ 7 files changed, 157 insertions(+), 3 deletions(-) diff --git a/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json b/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json index 7df3d5586..12de7e0d6 100644 --- a/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json +++ b/sdk/typescript/_bundled_plugin/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "codex-security", - "version": "0.1.27", + "version": "0.1.31", "description": "Codex Security workflows for security scans, analysis, and investigation.", "author": { "name": "OpenAI" diff --git a/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py b/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py index e997d1e5a..71b68709d 100644 --- a/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py +++ b/sdk/typescript/_bundled_plugin/scripts/deep_scan_workbench.py @@ -25,7 +25,7 @@ git_revision, worktree_content_digest, ) -from workbench_target_state import RegisteredRepositoryTarget +from workbench_target_state import RegisteredRepositoryTarget, require_scan_checkout_owner from workbench_validation import optional_text, require_uuid, user_text DEEP_SCAN_WORKER_KINDS = ("setup", "discovery", "dedup") @@ -620,6 +620,7 @@ def begin_deep_scan_for_scan( ) -> dict[str, Any]: scan_id = require_uuid(scan_id, "scan-id") candidate = require_scan(connection, scan_id) + require_scan_checkout_owner(connection, candidate) workspace = require_workspace(connection, candidate["workspace_id"]) if ( candidate["mode"] == "deep" @@ -681,6 +682,7 @@ def begin_deep_scan_for_scan( connection.execute("BEGIN IMMEDIATE") try: scan, _ = require_owned_scan(connection, scan_id, thread_id) + require_scan_checkout_owner(connection, scan) require_current_continuation( scan, args.claim_token, @@ -720,6 +722,7 @@ def begin_deep_scan_for_target( try: existing = existing_deep_scan_for_target(connection, thread_id, target_path, scope) if existing is not None: + require_scan_checkout_owner(connection, existing) existing_run = connection.execute( "SELECT 1 FROM deep_scan_runs WHERE scan_id = ?", (existing["id"],) ).fetchone() @@ -755,6 +758,7 @@ def begin_deep_scan_for_target( target_inode, ) if terminal is not None: + require_scan_checkout_owner(connection, terminal) connection.commit() return deep_scan_result( connection, diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index 4ce1af128..fa3a88ae9 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -123,6 +123,7 @@ backfill_security_targets, ensure_security_target, register_security_target, + require_scan_checkout_owner, ) from workbench_validation import ( bounded_output_text, @@ -821,6 +822,7 @@ def start_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict (workspace["id"],), ).fetchone() if active is not None: + require_scan_checkout_owner(connection, active) return workspace_state(connection, workspace["id"]) workspace_version = workspace["updated_at"] scan_id = str(uuid.uuid4()) @@ -865,6 +867,7 @@ def start_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict (workspace["id"],), ).fetchone() if active is not None: + require_scan_checkout_owner(connection, active) if manages_transaction: connection.commit() return workspace_state(connection, workspace["id"]) @@ -1021,6 +1024,7 @@ def _start_prompt_driven_scan( ), ).fetchone() if existing is not None: + require_scan_checkout_owner(connection, existing) connection.commit() return {**scan_context(connection, existing["id"]), "startDisposition": "joined"} target_root.mkdir(parents=True, exist_ok=True) diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py index 093c9d425..e7230a698 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_target_state.py @@ -633,6 +633,18 @@ def origin(self, state: RepositoryTargetState) -> tuple[str, str] | None: return self._origins[state.target_path] +def require_scan_checkout_owner( + connection: sqlite3.Connection, scan: sqlite3.Row | dict +) -> None: + identities = RepositoryIdentityCache(connection) + owner = identities.for_row(scan) + owner.require_owner() + if owner.missing or not identities.scope_for_scan(scan).contains(scan): + raise SystemExit( + "The saved scan no longer matches the selected repository checkout." + ) + + def _pre_release_repository_identities(identity: GitRepositoryIdentity) -> set[str]: directory = os.path.normcase(identity.common_directory) relative = os.path.normcase(os.fspath(Path(identity.relative_path))).replace(os.sep, "/") diff --git a/sdk/typescript/src/version.ts b/sdk/typescript/src/version.ts index 889395219..1da61e5e9 100644 --- a/sdk/typescript/src/version.ts +++ b/sdk/typescript/src/version.ts @@ -8,7 +8,7 @@ const PACKAGE_VERSIONS = packageVersions( export const VERSION = PACKAGE_VERSIONS.package; export const CODEX_SDK_VERSION = PACKAGE_VERSIONS.sdk; export const CODEX_EXECUTABLE_VERSION = PACKAGE_VERSIONS.executable; -export const BUNDLED_PLUGIN_VERSION = "0.1.27" as const; +export const BUNDLED_PLUGIN_VERSION = "0.1.31" as const; const PACKAGE_NAME = "@openai/codex-security"; const VERSION_PATTERN = diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts index b4a0469c8..dfc972709 100644 --- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts @@ -45,6 +45,7 @@ const deepScanOwnershipProbe = [ "elif case.get('mutation') == 'withdraw':", " connection.executescript(\"CREATE TRIGGER withdraw_handoff BEFORE UPDATE OF thread_id ON workspaces BEGIN UPDATE scans SET handoff_status = 'pending' WHERE workspace_id = NEW.id; END\")", "deep_scan.require_scan = lambda database, value: database.execute('SELECT * FROM scans WHERE id = ?', (value,)).fetchone()", + "deep_scan.require_scan_checkout_owner = lambda database, scan: None", "deep_scan.require_workspace = lambda database, value: database.execute('SELECT * FROM workspaces WHERE id = ?', (value,)).fetchone()", "deep_scan.now = lambda: 'after'", "deep_scan.deep_scan_result = lambda database, value, *, start_disposition=None: {'startDisposition': start_disposition}", diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts index 6f128c26f..ec247cac6 100644 --- a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -491,6 +491,106 @@ with ExitStack() as stack: "deepId": deep["scanId"], "registeredInsideTransaction": checks, "generationIndex": connection.execute("SELECT 1 FROM sqlite_master WHERE name = 'scans_by_repository_generation'").fetchone() is not None, })) + elif scenario == "scan-reuse": + for name, stored, live, generation in ( + ("current", "generation-current", None, "generation-current"), + ("legacy", None, "generation-legacy", None), + ("refused", "generation-before", "generation-after", "generation-before"), + ("contradictory", "generation-current", None, "generation-other"), + ("missing", "generation-missing", None, "generation-missing"), + ): + add_target(name, stored, live) + add_scan(name + "-scan", name, generation=generation, status="running") + missing.add(paths["missing"]) + connection.commit() + scan = lambda name: connection.execute( + "SELECT * FROM scans WHERE id = ?", (name + "-scan",) + ).fetchone() + accepted = {} + for name in paths: + try: + state.require_scan_checkout_owner(connection, scan(name)) + except SystemExit: + accepted[name] = False + else: + accepted[name] = True + + def reuse(name, kind): + target = Path(paths[name]) + scan_id = name + "-scan" + workspace_id = "workspace-" + scan_id + mode = "deep" if kind.startswith("deep-") else "standard" + headless = kind == "headless" + connection.execute( + "UPDATE workspaces SET thread_id = 'synthetic-thread', active_scan_id = ?, " + "default_scope = '.', default_mode = ?, submitted = 1 WHERE id = ?", + (scan_id, mode, workspace_id), + ) + connection.execute( + "UPDATE scans SET mode = ?, handoff_status = 'delivered', " + "handoff_claim_token = ?, continuation_thread_id = ? WHERE id = ?", + (mode, "synthetic-token" if headless else None, + "synthetic-thread" if headless else None, scan_id), + ) + connection.commit() + row = scan(name) + workspace = connection.execute( + "SELECT * FROM workspaces WHERE id = ?", (workspace_id,) + ).fetchone() + ensure_run = Mock() + with ExitStack() as mocks: + for module in (workbench, deep_workbench): + mocks.enter_context(patch.object(module, "require_uuid", side_effect=lambda value, label: value)) + mocks.enter_context(patch.object(module, "require_remediation_target", return_value=target)) + mocks.enter_context(patch.object(module, "directory_snapshot_regular_file_count", return_value=0)) + mocks.enter_context(patch.object(workbench, "workspace_state", return_value={})) + mocks.enter_context(patch.object(workbench, "scan_context", return_value={})) + mocks.enter_context(patch.object(workbench, "inspect_setup_values", return_value={ + "target": {"targetPath": str(target)}, "scope": ".", "diffTarget": None, + })) + mocks.enter_context(patch.object(workbench, "scan_target_identity", return_value=( + "synthetic", None, 7, metadata[str(target)].st_ino, + ))) + mocks.enter_context(patch.object(workbench, "scan_target_root", return_value=root / "artifacts")) + for function, value in { + "require_target": target, "require_scope": ".", + "require_scan": row, "require_workspace": workspace, + "require_owned_scan": (row, workspace), "git_revision": "synthetic", + "worktree_content_digest": "snapshot", "effective_deep_scan_config": {}, + "now": timestamp, "deep_scan_result": {}, + }.items(): + mocks.enter_context(patch.object(deep_workbench, function, return_value=value)) + mocks.enter_context(patch.object(deep_workbench, "require_scannable_target")) + mocks.enter_context(patch.object(deep_workbench, "require_current_continuation")) + mocks.enter_context(patch.object(deep_workbench, "ensure_deep_scan_run", ensure_run)) + existing = [row] if kind == "deep-direct" else [None, row] if kind == "deep-transaction" else [None, None] + mocks.enter_context(patch.object(deep_workbench, "existing_deep_scan_for_target", side_effect=existing)) + mocks.enter_context(patch.object(deep_workbench, "terminal_deep_scan_for_target_snapshot", return_value=row)) + args = argparse.Namespace( + workspace_id=workspace_id, target_path=str(target), scope=".", mode="standard", + thread_id="synthetic-thread", diff_target_kind=None, diff_base_revision=None, + diff_head_revision=None, diff_content_digest=None, user_context=None, + target_summary=None, scan_root=None, model=None, reasoning_effort=None, + claim_token=None, workflow_version="synthetic", + ) + try: + if kind == "workspace": + workbench.start_scan(connection, args) + elif kind in ("prompt", "headless"): + workbench._start_prompt_driven_scan(connection, args, headless_standard=headless) + else: + deep_workbench.begin_deep_scan_for_target(connection, args, "synthetic-thread") + except SystemExit: + return {"accepted": False, "createdRun": ensure_run.call_count != 0} + return {"accepted": True, "createdRun": ensure_run.call_count != 0} + + kinds = ("workspace", "prompt", "headless", "deep-direct", "deep-transaction", "deep-terminal") + print(json.dumps({ + "owners": accepted, + "reuse": {name: {kind: reuse(name, kind) for kind in kinds} + for name in ("current", "refused")}, + "legacyGeneration": scan("legacy")["repository_generation"], + })) elif scenario == "birth-time": path = "/synthetic repository/.git" real_sizeof = ctypes.sizeof @@ -1932,6 +2032,39 @@ test("revalidates saved workspace identity inside the scan-start transaction", ( expect(result["stored"]).toBe("repository-current"); }); +test("checks current ownership before rejoining saved scan tasks", () => { + const result = run("scan-reuse"); + + expect(result["owners"]).toEqual({ + current: true, + legacy: true, + refused: false, + contradictory: false, + missing: false, + }); + expect(result["reuse"]).toEqual({ + current: { + workspace: { accepted: true, createdRun: false }, + prompt: { accepted: true, createdRun: false }, + headless: { accepted: true, createdRun: false }, + "deep-direct": { accepted: true, createdRun: true }, + "deep-transaction": { accepted: true, createdRun: true }, + "deep-terminal": { accepted: true, createdRun: false }, + }, + refused: Object.fromEntries( + [ + "workspace", + "prompt", + "headless", + "deep-direct", + "deep-transaction", + "deep-terminal", + ].map((kind) => [kind, { accepted: false, createdRun: false }]), + ), + }); + expect(result["legacyGeneration"]).toBeNull(); +}); + test("quarantines unproved public-v30 bindings without discarding historical records", () => { const result = run("v30-current");