diff --git a/.github/workflows/reusable.yml b/.github/workflows/reusable.yml index 01146a0..7db00d8 100644 --- a/.github/workflows/reusable.yml +++ b/.github/workflows/reusable.yml @@ -55,9 +55,16 @@ name: reusable # could only see one skill would have to miss. # # The caller maps one secret onto `api_key`, so this workflow never learns -# your provider, your gateway, or anything else in the vault. `secrets: inherit` -# still works if you would rather pass the vault and name the key with -# `api_key_secret`; it is more than this workflow needs. +# anything else in the vault. `secrets: inherit` still works if you would rather +# pass the vault and name the key with `api_key_secret`; it is more than this +# workflow needs. A key issued by a gateway rather than the provider also needs +# `api_base_url`, and `api_custom_headers` if the gateway reads the key from a +# header of its own: +# +# with: +# api_base_url: https://gateway.example.com/anthropic +# api_custom_headers: | +# Ocp-Apim-Subscription-Key: $API_KEY # # The `uses:` pin on this workflow is the harness that runs. GitHub does not # let a reusable workflow interpolate that pin into `uses: amd/skillscope@...`, @@ -113,6 +120,20 @@ on: Blank means the runner is already authenticated and no key is exported. type: string default: ANTHROPIC_API_KEY + api_base_url: + description: >- + ANTHROPIC_BASE_URL for the routing and behavioral legs, for a key that + belongs to a gateway rather than to the provider. Blank uses the + provider default. + type: string + default: "" + api_custom_headers: + description: >- + ANTHROPIC_CUSTOM_HEADERS for the routing and behavioral legs, one + `Name: value` per line. `$API_KEY` is replaced with the key, so a + gateway that wants the key in its own header needs no second secret. + type: string + default: "" secrets: api_key: @@ -298,24 +319,12 @@ jobs: ref: ${{ job.workflow_sha }} path: .skillscope-action - - name: Check the model credentials - shell: python + - name: Resolve the model credentials env: EXPECT_KEY: ${{ inputs.routing != 'off' && (inputs.api_key_secret != '' || secrets.api_key != '') }} - run: | - import os - import sys - - if os.environ.get("EXPECT_KEY") != "true": - sys.exit(0) - if not os.environ.get("ANTHROPIC_API_KEY", "").strip(): - sys.exit( - "error: the model API key resolved to an empty value. GitHub " - "withholds secrets from pull requests opened from a fork, so " - "re-run this from a branch in the repository; otherwise check " - "that the secret is set and that the caller maps it onto " - "'api_key' (or passes 'secrets: inherit')." - ) + API_BASE_URL: ${{ inputs.api_base_url }} + API_CUSTOM_HEADERS: ${{ inputs.api_custom_headers }} + run: python .skillscope-action/skillscope/credentials.py --reusable - name: Run the routing eval for ${{ matrix.skill }} uses: ./.skillscope-action @@ -371,24 +380,12 @@ jobs: ref: ${{ job.workflow_sha }} path: .skillscope-action - - name: Check the model credentials - shell: python + - name: Resolve the model credentials env: EXPECT_KEY: ${{ inputs.behavioral != 'off' && (inputs.api_key_secret != '' || secrets.api_key != '') }} - run: | - import os - import sys - - if os.environ.get("EXPECT_KEY") != "true": - sys.exit(0) - if not os.environ.get("ANTHROPIC_API_KEY", "").strip(): - sys.exit( - "error: the model API key resolved to an empty value. GitHub " - "withholds secrets from pull requests opened from a fork, so " - "re-run this from a branch in the repository; otherwise check " - "that the secret is set and that the caller maps it onto " - "'api_key' (or passes 'secrets: inherit')." - ) + API_BASE_URL: ${{ inputs.api_base_url }} + API_CUSTOM_HEADERS: ${{ inputs.api_custom_headers }} + run: python .skillscope-action/skillscope/credentials.py --reusable - name: Run the behavioral cases for ${{ matrix.skill }} uses: ./.skillscope-action diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index 49fc4de..78c2a5f 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -533,7 +533,6 @@ jobs: - name: Resolve model credentials if: inputs.api_key_secret != '' || inputs.federation_rule_id != '' - shell: python env: API_KEY: ${{ secrets[inputs.api_key_secret] }} API_BASE_URL: ${{ inputs.api_base_url }} @@ -544,14 +543,7 @@ jobs: FEDERATION_SERVICE_ACCOUNT_ID: ${{ inputs.federation_service_account_id }} FEDERATION_WORKSPACE_ID: ${{ inputs.federation_workspace_id }} FEDERATION_AUDIENCE: ${{ inputs.federation_audience }} - run: | - import subprocess - import sys - - completed = subprocess.run( - [sys.executable, ".skillscope-action/skillscope/credentials.py"] - ) - raise SystemExit(completed.returncode) + run: python .skillscope-action/skillscope/credentials.py - name: Run the routing eval uses: ./.skillscope-action @@ -619,7 +611,6 @@ jobs: - name: Resolve model credentials if: inputs.api_key_secret != '' || inputs.federation_rule_id != '' - shell: python env: API_KEY: ${{ secrets[inputs.api_key_secret] }} API_BASE_URL: ${{ inputs.api_base_url }} @@ -630,14 +621,7 @@ jobs: FEDERATION_SERVICE_ACCOUNT_ID: ${{ inputs.federation_service_account_id }} FEDERATION_WORKSPACE_ID: ${{ inputs.federation_workspace_id }} FEDERATION_AUDIENCE: ${{ inputs.federation_audience }} - run: | - import subprocess - import sys - - completed = subprocess.run( - [sys.executable, ".skillscope-action/skillscope/credentials.py"] - ) - raise SystemExit(completed.returncode) + run: python .skillscope-action/skillscope/credentials.py - name: Run behavioral cases for ${{ matrix.skill }} uses: ./.skillscope-action @@ -683,7 +667,6 @@ jobs: - name: Resolve model credentials if: inputs.scoped_api_key_secret != '' || inputs.scoped_federation_rule_id != '' - shell: python env: API_KEY: ${{ secrets[inputs.scoped_api_key_secret] }} API_BASE_URL: ${{ inputs.scoped_api_base_url }} @@ -695,14 +678,7 @@ jobs: FEDERATION_SERVICE_ACCOUNT_ID: ${{ inputs.scoped_federation_service_account_id }} FEDERATION_WORKSPACE_ID: ${{ inputs.scoped_federation_workspace_id }} FEDERATION_AUDIENCE: ${{ inputs.federation_audience }} - run: | - import subprocess - import sys - - completed = subprocess.run( - [sys.executable, ".skillscope-action/skillscope/credentials.py"] - ) - raise SystemExit(completed.returncode) + run: python .skillscope-action/skillscope/credentials.py - name: Run behavioral cases for ${{ matrix.skill }} uses: ./.skillscope-action diff --git a/docs/usage.md b/docs/usage.md index 68164ef..17cd0f9 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -254,6 +254,25 @@ Naming several, and holding them to different bars: | `min_accuracy` | `1` | The routing bar. `0` reports the score without gating on it. | | `api_key` | (none) | The model API key, mapped from the caller's vault. One secret, not the whole set. | | `api_key_secret` | `ANTHROPIC_API_KEY` | Name to look up under `secrets: inherit`, if you would rather pass the vault than map one key. | +| `api_base_url` | (none) | Base URL of the gateway that issued the key. Blank talks to the provider directly. | +| `api_custom_headers` | (none) | Extra headers for that gateway, one `Name: value` per line. `$API_KEY` is replaced with the key. | + +A key issued by an API gateway rather than by the provider needs the gateway's +address, and usually the key in a header of its own: + +```yaml + secrets: + api_key: ${{ secrets.GATEWAY_API_KEY }} + with: + skills: path/to/my-skill + api_base_url: https://gateway.example.com/anthropic + api_custom_headers: | + Ocp-Apim-Subscription-Key: $API_KEY +``` + +Setting either input without a key stops the run with an error rather than +sending unauthenticated requests. Put the key in a header as `$API_KEY`, never +as a literal: GitHub only masks values it received as secrets. `optional` is for a bar you have not met yet: the leg runs, the report lands in the step summary, and a red leg leaves the run green. `off` does not run it at diff --git a/skillscope/credentials.py b/skillscope/credentials.py index 6bbff3e..b1f064a 100644 --- a/skillscope/credentials.py +++ b/skillscope/credentials.py @@ -185,12 +185,46 @@ def resolve(env: Mapping[str, str], *, fetch: Fetch | None = None) -> dict[str, return exported +def resolve_for_reusable(env: Mapping[str, str]) -> dict[str, str]: + """What a reusable.yml job exports. Its key is already in the job's + environment, so only a gateway gives it anything to add.""" + base_url = env.get("API_BASE_URL", "").strip() + headers = env.get("API_CUSTOM_HEADERS", "").strip() + if env.get("EXPECT_KEY") != "true": + if base_url or headers: + raise CredentialError( + "api_base_url and api_custom_headers are for a gateway key, but " + "no key is configured. Map the key onto 'api_key', or name it " + "with 'api_key_secret'." + ) + return {} + + key = env.get("ANTHROPIC_API_KEY", "").strip() + if not key: + raise CredentialError( + "the model API key resolved to an empty value. GitHub withholds " + "secrets from pull requests opened from a fork, so re-run this from " + "a branch in the repository; otherwise check that the secret is set " + "and that the caller maps it onto 'api_key' (or passes " + "'secrets: inherit')." + ) + if not base_url and not headers: + return {} + return resolve( + {"API_KEY": key, "API_BASE_URL": base_url, "API_CUSTOM_HEADERS": headers} + ) + + def main(argv: list[str] | None = None) -> int: + argv = sys.argv[1:] if argv is None else argv + pick = resolve_for_reusable if "--reusable" in argv else resolve try: - exported = resolve(os.environ) + exported = pick(os.environ) except CredentialError as error: print(f"error: {error}", file=sys.stderr) return 1 + if not exported: + return 0 # Minted during the run, so GitHub has never seen it and will not redact it. if "ANTHROPIC_AUTH_TOKEN" in exported: diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 11cdee6..0bf0fb1 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -212,9 +212,10 @@ class TestEveryRunStepIsPortable(unittest.TestCase): Which platforms a run lands on is the caller's to decide -- `runner` here, `behavior_os` and a skill's own `machine.yml` in the full pipeline -- so - every step that can reach one of them is written in Python. That is not a - taste in scripting languages; it is the only shell all three platforms are - guaranteed to agree on, and this test is what keeps the next step honest. + every step that can reach one of them either names Python as its shell or, + in a workflow, names none. That is not a taste in scripting languages; it + is the only shell all three platforms are guaranteed to agree on, and this + test is what keeps the next step honest. """ # Every runner in these three is a caller's input -- `runner`, @@ -243,10 +244,12 @@ def steps(self, document: object) -> list[dict]: def test_no_run_step_names_a_shell_a_runner_might_not_have(self) -> None: import yaml - portable = {"python"} for relative in self.CI_FILES: path = REPO_ROOT / relative self.assertTrue(path.is_file(), f"{relative} is missing") + # A workflow step may leave `shell:` unset and get the runner's + # default; a composite action step has no default to fall back on. + portable = {"python"} if relative.name == "action.yml" else {"python", ""} document = yaml.safe_load(path.read_text(encoding="utf-8")) for step in self.steps(document): if "run" not in step: @@ -926,6 +929,38 @@ def test_the_minted_token_is_masked_before_anything_can_log_it(self) -> None: self.assertIn("sk-ant-oat01-minted", written) self.assertNotIn("ANTHROPIC_API_KEY", written) + def test_a_gateway_key_reaches_the_env_file_but_never_stdout(self) -> None: + # GitHub masks the key only because it arrived as a secret, so nothing + # printed may carry it, substituted into a header or otherwise. + with tempfile.TemporaryDirectory() as tmp: + github_env = Path(tmp) / "github.env" + github_env.touch() + environment = { + "EXPECT_KEY": "true", + "ANTHROPIC_API_KEY": "the-gateway-key", + "API_BASE_URL": "https://gateway.example/anthropic", + "API_CUSTOM_HEADERS": "Ocp-Apim-Subscription-Key: $API_KEY", + "GITHUB_ENV": str(github_env), + } + stdout, stderr = io.StringIO(), io.StringIO() + with mock.patch.dict(os.environ, environment, clear=True), contextlib.redirect_stdout( + stdout + ), contextlib.redirect_stderr(stderr): + self.assertEqual(credentials.main(["--reusable"]), 0) + + self.assertNotIn("the-gateway-key", stdout.getvalue() + stderr.getvalue()) + written = github_env.read_text(encoding="utf-8") + self.assertIn("ANTHROPIC_BASE_URL<<", written) + self.assertIn("Ocp-Apim-Subscription-Key: the-gateway-key", written) + + def test_a_gateway_without_a_key_is_refused(self) -> None: + with self.assertRaises(credentials.CredentialError) as raised: + credentials.resolve_for_reusable( + {"EXPECT_KEY": "false", "API_BASE_URL": "https://gateway.example"} + ) + self.assertIn("no key is configured", str(raised.exception)) + self.assertEqual(credentials.resolve_for_reusable({"EXPECT_KEY": "false"}), {}) + def test_running_credentials_as_a_script_does_not_shadow_stdlib_select(self) -> None: # Graded jobs run this file by path, which puts the package directory on # sys.path. A module named select.py would shadow the stdlib and this