From fe4c6945ed5c5e128ac035c1c045213db1cbd749 Mon Sep 17 00:00:00 2001 From: Tobias Stenzel Date: Tue, 28 Jul 2026 17:27:02 +0200 Subject: [PATCH 1/2] version 2026.7.28 --- src/appenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/appenv.py b/src/appenv.py index ea76864..20f9b29 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -15,7 +15,7 @@ from __future__ import annotations -__version__ = "2026.6.30" +__version__ = "2026.7.28" import argparse import difflib From 8944c1dc2ca0398ef85d1ad5ed672c6d179ff253 Mon Sep 17 00:00:00 2001 From: Tobias Stenzel Date: Fri, 31 Jul 2026 09:16:30 +0200 Subject: [PATCH 2/2] migrate: improve migration resilience, version 2026.7.29 - Keep index_url credentials migrated from requirements.txt. User just gets a warning and can decide. Before, removing credentials broke `uv lock` because packages couldn't be resolved. - Move ensure_gitignore before uv lock so .gitignore survives lock crashes. - Trim verbose docstrings, inline small single-use functions --- docs/user/commands.md | 12 +++- src/appenv.py | 95 ++++++++++------------------- src/appenv.pyi | 3 +- tests/test_gitignore_consistency.py | 38 ++++++++++++ tests/test_migrate_pip_options.py | 55 +++++++++-------- 5 files changed, 111 insertions(+), 92 deletions(-) diff --git a/docs/user/commands.md b/docs/user/commands.md index c1b0068..8a57a79 100644 --- a/docs/user/commands.md +++ b/docs/user/commands.md @@ -150,14 +150,22 @@ Every line starting with `-` is treated as a pip-option and kept out of the `[pr `--index-url` and `--extra-index-url` are translated into [`[[tool.uv.index]]`](https://docs.astral.sh/uv/configuration/indexes/) entries in `pyproject.toml`. This is what unblocks projects using private registries (GitLab Package Registry, AWS CodeArtifact, Artifactory). -**Credentials are never copied into `pyproject.toml`.** If an index URL carries embedded credentials (deploy token, API key), migrate strips them and writes only the clean URL, then tells you which environment variables to set so `uv lock` can authenticate. The names follow uv's [index authentication](https://docs.astral.sh/uv/configuration/indexes/#authenticated-private-registries) convention: +**Credentials are preserved in `pyproject.toml` so `uv lock` can authenticate +immediately** — if the index URL carries embedded credentials (deploy token, +API key), they are written as-is to the `url` field in +`[[tool.uv.index]]`. This is an accepted security trade-off: the secret is +needed for package resolution. However, the credential is removed from +standard output and the warning suggests switching to uv's index environment +variables to keep secrets out of VCS: | Variable | Purpose | |----------|---------| | `UV_INDEX__USERNAME` | Username for index `` | | `UV_INDEX__PASSWORD` | Password or token for index `` | -`` is the index name in uppercase. Set these in your shell (or CI secrets) before running `uv lock`. As an alternative to environment variables, you can put credentials in `~/.netrc`. +`` is the index name in uppercase. After migration, consider removing +the embedded credentials from `pyproject.toml` and setting these variables in +your shell or CI secrets instead. #### Unsupported options → dropped with warning diff --git a/src/appenv.py b/src/appenv.py index 20f9b29..2ae2098 100755 --- a/src/appenv.py +++ b/src/appenv.py @@ -15,7 +15,7 @@ from __future__ import annotations -__version__ = "2026.7.28" +__version__ = "2026.7.29" import argparse import difflib @@ -111,14 +111,11 @@ def error(self, message: str) -> NoReturn: class IndexEntry(NamedTuple): - """A ``[[tool.uv.index]]`` entry migrated from a pip index option. - - ``url`` is always credential-free: the literal secret is stripped before - this object is constructed and never reaches pyproject.toml or stdout. - """ + """A migrated ``[[tool.uv.index]]`` entry; ``url`` is ``raw_url`` minus credentials.""" name: str url: str + raw_url: str has_credentials: bool @@ -158,37 +155,6 @@ def _split_pip_option_token(token: str) -> tuple[str, str | None]: return name, value if sep else None -def _strip_index_url_credentials(url: str) -> tuple[str, bool]: - """Return ``(clean_url, has_credentials)`` for a pip index URL. - - Strips any ``user:password@`` userinfo so the literal secret can never - reach pyproject.toml. A credential-free URL is returned verbatim so it - round-trips exactly (e.g. ``==`` against the original holds). - - Works on the raw netloc string rather than :class:`urllib.parse.SplitResult` - properties, which raise ``ValueError`` on malformed ports. - """ - parts = urlsplit(url) - netloc = parts.netloc - if "@" not in netloc: - return url, False - userinfo, _, hostpart = netloc.rpartition("@") - clean_netloc = hostpart - clean = urlunsplit( - (parts.scheme, clean_netloc, parts.path, parts.query, parts.fragment) - ) - return clean, bool(userinfo.strip()) - - -def _uv_index_env_token(name: str) -> str: - """uv env-var token for an index name: uppercase, non-alnum -> ``_``. - - ``UV_INDEX__USERNAME`` / ``_PASSWORD`` per - https://docs.astral.sh/uv/concepts/indexes/#authentication - """ - return re.sub(r"[^A-Z0-9]", "_", name.upper()) - - def _index_name_from_url(url: str, used_names: set[str]) -> str: """Derive a unique, uv-compatible index name from the URL host. @@ -207,16 +173,22 @@ def _index_name_from_url(url: str, used_names: set[str]) -> str: def _build_index_entries(raw_urls: list[str]) -> list[IndexEntry]: - """Build de-duplicated, credential-free index entries from raw URLs. - - Duplicate URLs (after credential stripping) collapse to one entry, matching - pip's own de-duplication of repeated ``--extra-index-url`` lines. - """ + """De-duplicated index entries from raw pip index URLs.""" entries: list[IndexEntry] = [] used_names: set[str] = set() seen_urls: set[str] = set() for raw in raw_urls: - clean, has_credentials = _strip_index_url_credentials(raw) + parts = urlsplit(raw) + netloc = parts.netloc + if "@" in netloc: + userinfo, _, host = netloc.rpartition("@") + clean = urlunsplit( + (parts.scheme, host, parts.path, parts.query, parts.fragment) + ) + has_credentials = bool(userinfo.strip()) + else: + clean = raw + has_credentials = False if clean in seen_urls: continue seen_urls.add(clean) @@ -224,6 +196,7 @@ def _build_index_entries(raw_urls: list[str]) -> list[IndexEntry]: IndexEntry( name=_index_name_from_url(clean, used_names), url=clean, + raw_url=raw, has_credentials=has_credentials, ) ) @@ -288,10 +261,9 @@ def _parse_requirement_line( ) -> tuple[str | None, list[str], list[str]]: """Parse one requirements.txt line into ``(dep, index_urls, skipped)``. - ``dep`` is the dependency with all inline pip-options removed, or ``None`` - when the line carried only options. ``index_urls`` are raw index URLs - (credentials stripped later, collectively). ``skipped`` lists each dropped - option by name (may contain duplicates across ``--hash`` repeats). + ``dep`` is the dependency with inline pip-options removed, or ``None`` + when the line carried only options. ``skipped`` may contain duplicates + (e.g. repeated ``--hash`` on the same line). """ tokens = line.split() dep_tokens: list[str] = [] @@ -544,9 +516,9 @@ def print_migration_info(self) -> None: index.url, ) continue - token = _uv_index_env_token(index.name) + token = re.sub(r"[^A-Z0-9]", "_", index.name.upper()) log.warning( - "migration-index-credentials-stripped: name=%s url=%s " + "migration-index-credentials-present: name=%s url=%s " "env_username=UV_INDEX_%s_USERNAME env_password=UV_INDEX_%s_PASSWORD", index.name, index.url, @@ -554,13 +526,12 @@ def print_migration_info(self) -> None: token, ) print( - f"Warning: index '{index.name}' had embedded credentials in " - f"requirements.txt. They are NOT stored in pyproject.toml." + f"WARNING: index '{index.name}' has embedded credentials " + f"in pyproject.toml — they work now but may leak secrets to VCS." ) print( - f"Provide them via environment variables " - f"UV_INDEX_{token}_USERNAME and UV_INDEX_{token}_PASSWORD " - f"(or a ~/.netrc entry for {index.url})." + f"Consider replacing them with environment variables " + f"UV_INDEX_{token}_USERNAME and UV_INDEX_{token}_PASSWORD." ) print() @@ -588,14 +559,7 @@ def print_migration_info(self) -> None: ) def _parse_requirements_file(self) -> RequirementsTxtInfo: - """Parse requirements.txt into dependencies, indexes, and skipped options. - - pip-options (``--index-url``, ``--hash``, ...) are separated from real - dependencies so they never leak into ``[project] dependencies`` as - invalid PEP 508 specifiers. Index URLs become :class:`IndexEntry` - objects (credentials stripped); unsupported options are recorded in - ``skipped_options`` for the migrate warning. - """ + """Parse requirements.txt into :class:`RequirementsTxtInfo`.""" log.debug("parse-requirements-file: path=%s", self.requirements_path) content = self.requirements_path.read_text() raw_lines = [ @@ -706,7 +670,7 @@ def _generate_content_with_project( def _uv_index_section(indexes: list[IndexEntry]) -> str: """Render ``[[tool.uv.index]]`` entries from migrated index options.""" blocks = [ - f'[[tool.uv.index]]\nname = "{index.name}"\nurl = "{index.url}"' + f'[[tool.uv.index]]\nname = "{index.name}"\nurl = "{index.raw_url}"' for index in indexes ] return "\n\n".join(blocks) + "\n" @@ -1880,6 +1844,10 @@ def migrate( pyproject = pyproject.migrate_from_requirements_txt() pyproject.print_migration_info() + # Write .gitignore BEFORE uv lock so the ignore entries survive a + # lock failure and users are not suprised by untracked files. + ensure_gitignore(self.base, _GITIGNORE_ENTRIES) + uv_lock_out = self._uv_lock(uv, diff=False) print(uv_lock_out) @@ -1887,7 +1855,6 @@ def migrate( print("Preparing/cleaning .appenv directory ...") self._prepare_appenv_dir() - ensure_gitignore(self.base, _GITIGNORE_ENTRIES) log.info("migrate-completed: base=%s", self.base) print("\n=== Pyproject Migration completed ===") print("requirements.{txt,lock} kept as legacy. You can delete these files now.") diff --git a/src/appenv.pyi b/src/appenv.pyi index 5010c7d..ed3f93e 100644 --- a/src/appenv.pyi +++ b/src/appenv.pyi @@ -30,6 +30,7 @@ class UsageArgumentParser(argparse.ArgumentParser): class IndexEntry(NamedTuple): name: str url: str + raw_url: str has_credentials: bool _PIP_INDEX_URL_OPTIONS: Final[frozenset[str]] @@ -44,8 +45,6 @@ class RequirementsTxtInfo(NamedTuple): skipped_options: list[str] def _split_pip_option_token(token: str) -> tuple[str, str | None]: ... -def _strip_index_url_credentials(url: str) -> tuple[str, bool]: ... -def _uv_index_env_token(name: str) -> str: ... def _index_name_from_url(url: str, used_names: set[str]) -> str: ... def _build_index_entries(raw_urls: list[str]) -> list[IndexEntry]: ... def _consume_next_value( diff --git a/tests/test_gitignore_consistency.py b/tests/test_gitignore_consistency.py index fbd56e3..3d48d8f 100644 --- a/tests/test_gitignore_consistency.py +++ b/tests/test_gitignore_consistency.py @@ -9,6 +9,10 @@ command never silently commits the `.appenv/venv/` tree. """ +import subprocess + +import pytest + import appenv EXPECTED_GITIGNORE_ENTRIES = appenv._GITIGNORE_ENTRIES @@ -136,3 +140,37 @@ def capture(base, entries): ) assert init_entries is appenv._GITIGNORE_ENTRIES assert init_entries == EXPECTED_GITIGNORE_ENTRIES + + +def test_migrate_writes_gitignore_even_when_uv_lock_fails( + tmp_path, monkeypatch, app_env +): + """reorder-migrate::gitignore-before-lock — ensure_gitignore runs BEFORE uv + lock, so .gitignore is written even when uv lock crashes. + + Regression test for the original bug: migrate() called ensure_gitignore + AFTER _uv_lock, so a lock failure (e.g. missing credentials for a private + index) prevented .gitignore from being written. + """ + base = tmp_path + (base / "requirements.txt").write_text( + "--extra-index-url https://deploy:s3cr3t@gitlab.example.com/simple\nrequests\n" + ) + + def _raise(self, uv, diff=False): + raise subprocess.CalledProcessError(returncode=1, cmd=["uv", "lock"]) + + monkeypatch.setattr(appenv.AppEnv, "_uv_lock", _raise) + + env = app_env(base) + with pytest.raises(subprocess.CalledProcessError): + env.migrate() + + # .gitignore was written despite the crash. + gitignore = base / ".gitignore" + assert gitignore.exists() + gitignore_lines = gitignore.read_text().splitlines() + for entry in EXPECTED_GITIGNORE_ENTRIES: + assert entry in gitignore_lines, ( + f"migrate must add {entry!r} to .gitignore even when uv lock fails" + ) diff --git a/tests/test_migrate_pip_options.py b/tests/test_migrate_pip_options.py index c8c9749..6b6d69a 100644 --- a/tests/test_migrate_pip_options.py +++ b/tests/test_migrate_pip_options.py @@ -10,20 +10,20 @@ What is pinned (decided by the spec): * pip-options are separated from real dependencies and never written as deps. * ``--index-url`` / ``--extra-index-url`` become ``[[tool.uv.index]]`` entries. - * embedded credentials are stripped (the literal secret never reaches - pyproject.toml or stdout) and the user is warned with the exact - ``UV_INDEX__USERNAME`` / ``_PASSWORD`` env var names, where ```` - is derived from the index name per uv's documented mechanism - (https://docs.astral.sh/uv/concepts/indexes/#authentication). + * embedded credentials are kept in pyproject.toml (so ``uv lock`` succeeds + without manual env-var setup), but the user is warned with the exact + ``UV_INDEX__USERNAME`` / ``_PASSWORD`` env var names and told to + consider removing the credentials from the URL. * options without a pyproject equivalent (``--hash``, ``--no-binary``, ``--only-binary``, ``--require-hashes``) are dropped with a warning that lists the skipped options. Deliberately NOT pinned (spec leaves open, so the implementer chooses): - * how the index ``name`` is derived from the URL. - * the exact stored URL representation when credentials are present (clean URL - vs ``$VAR`` placeholder). Only the security guarantee (no literal secret - leak) and the env-var guidance are asserted. + * how the index ``name`` is derived from the URL. + * the exact stored URL representation when credentials are present. + The security guarantee (no literal secret leak to stdout) and the + env-var guidance are asserted. Credentials in pyproject.toml are an + accepted trade-off — the warning tells the user to remove them. """ import re @@ -185,17 +185,20 @@ def test_migrate_assigns_distinct_names_to_multiple_indexes(migrate_reqs): # ============================================================================ -# Credentials: stripped (no literal secret leak) + env-var guidance. +# Credentials: kept in pyproject (so uv lock works), warned to stdout. # ============================================================================ -def test_migrate_strips_index_credentials_and_warns_env_vars(migrate_reqs, patterns): - """Credentialed index URL: the secret never reaches pyproject or stdout, and - the user is told the exact ``UV_INDEX__USERNAME`` / ``_PASSWORD`` names. +def test_migrate_keeps_credentials_in_pyproject_and_warns_env_vars( + migrate_reqs, patterns +): + """Credentialed index URL: the secret IS kept in pyproject.toml (so ``uv + lock`` works without manual env-var setup), but the user is warned with + the exact ``UV_INDEX__USERNAME`` / ``_PASSWORD`` names and instructed + to consider removing credentials from the URL. - Merges the security contract (no literal leak anywhere) with the UX contract - (env-var guidance emitted). The exact derived token is still asserted per - index so a token-derivation regression cannot hide behind a wildcard. + Security contract: no literal secret leak to stdout (pyproject.toml is the + accepted trade-off — the warning tells the user to fix it). """ result = migrate_reqs( "--extra-index-url https://deploy:s3cr3t@gitlab.example.com/simple\nrequests\n" @@ -205,8 +208,8 @@ def test_migrate_strips_index_credentials_and_warns_env_vars(migrate_reqs, patte assert len(indexes) == 1 assert "gitlab.example.com" in indexes[0]["url"] - # Security: the literal secret must never appear in pyproject. - assert "s3cr3t" not in result.pyproject + # Secret is in pyproject.toml so uv lock can authenticate. + assert "s3cr3t" in result.pyproject # UX: the exact derived env-var token is advertised for each credentialed index. for idx in indexes: @@ -308,7 +311,11 @@ def test_migrate_warns_lists_all_skipped_pip_options(migrate_reqs): def test_migrate_gitlab_private_registry_end_to_end(migrate_reqs): - """A GitLab Package Registry requirements.txt migrates cleanly.""" + """A GitLab Package Registry requirements.txt migrates cleanly. + + Credentials are kept in pyproject (so uv lock works) but not in stdout; + the user is warned to replace them with env vars. + """ result = migrate_reqs( "--extra-index-url https://deploy:s3cr3t@gitlab.example.com/api/v4/" "projects/42/packages/pypi/simple\n" @@ -323,14 +330,14 @@ def test_migrate_gitlab_private_registry_end_to_end(migrate_reqs): "requests", ] - # Index entry created, host preserved, secret stripped everywhere. + # Index entry created, host preserved, secret kept in pyproject (so uv + # lock works), not leaked to stdout. indexes = _extract_uv_indexes(result.pyproject) assert len(indexes) == 1 assert ( "gitlab.example.com/api/v4/projects/42/packages/pypi/simple" in indexes[0]["url"] ) - assert "s3cr3t" not in result.pyproject assert "s3cr3t" not in result.stdout # Credential env-var guidance + skipped-option warnings are emitted. @@ -405,9 +412,9 @@ def test_migrate_assigns_numeric_suffix_when_index_hosts_collide(migrate_reqs): def test_migrate_dedups_repeated_identical_index_url(migrate_reqs): """Two identical --extra-index-url lines collapse to a single index entry. - Drives the de-duplication ``continue`` branch: a URL already seen (after - credential stripping) is not emitted a second time, matching pip's own - de-duplication of repeated ``--extra-index-url`` lines. + Drives the de-duplication ``continue`` branch: a URL already seen (the + dedup key is the credential-free ``url``) is not emitted a second time, + matching pip's own de-duplication of repeated ``--extra-index-url`` lines. """ result = migrate_reqs( "--extra-index-url https://gitlab.example.com/simple\n"