migrate: improve migration resilience, version 2026.7.29

docs/user/commands.md

@@ -150,14 +150,22 @@ Every line starting with `-` is treated as a pip-option and kept out of the `[pr
 
 `--index-url` and `--extra-index-url` are translated into [`[[tool.uv.index]]`](https://docs.astral.sh/uv/configuration/indexes/) entries in `pyproject.toml`. This is what unblocks projects using private registries (GitLab Package Registry, AWS CodeArtifact, Artifactory).
 
-**Credentials are never copied into `pyproject.toml`.** If an index URL carries embedded credentials (deploy token, API key), migrate strips them and writes only the clean URL, then tells you which environment variables to set so `uv lock` can authenticate. The names follow uv's [index authentication](https://docs.astral.sh/uv/configuration/indexes/#authenticated-private-registries) convention:
+**Credentials are preserved in `pyproject.toml` so `uv lock` can authenticate
+immediately** — if the index URL carries embedded credentials (deploy token,
+API key), they are written as-is to the `url` field in
+`[[tool.uv.index]]`. This is an accepted security trade-off: the secret is
+needed for package resolution. However, the credential is removed from
+standard output and the warning suggests switching to uv's index environment
+variables to keep secrets out of VCS:
 
 | Variable | Purpose |
 |----------|---------|
 | `UV_INDEX_<NAME>_USERNAME` | Username for index `<NAME>` |
 | `UV_INDEX_<NAME>_PASSWORD` | Password or token for index `<NAME>` |
 
-`<NAME>` is the index name in uppercase. Set these in your shell (or CI secrets) before running `uv lock`. As an alternative to environment variables, you can put credentials in `~/.netrc`.
+`<NAME>` is the index name in uppercase. After migration, consider removing
+the embedded credentials from `pyproject.toml` and setting these variables in
+your shell or CI secrets instead.
 
 #### Unsupported options → dropped with warning
 

src/appenv.py

@@ -15,7 +15,7 @@
 
 from __future__ import annotations
 
-__version__ = "2026.6.30"
+__version__ = "2026.7.29"
 
 import argparse
 import difflib
@@ -111,14 +111,11 @@ def error(self, message: str) -> NoReturn:
 
 
 class IndexEntry(NamedTuple):
-    """A ``[[tool.uv.index]]`` entry migrated from a pip index option.
-
-    ``url`` is always credential-free: the literal secret is stripped before
-    this object is constructed and never reaches pyproject.toml or stdout.
-    """
+    """A migrated ``[[tool.uv.index]]`` entry; ``url`` is ``raw_url`` minus credentials."""
 
     name: str
     url: str
+    raw_url: str
     has_credentials: bool
 
 
@@ -158,37 +155,6 @@ def _split_pip_option_token(token: str) -> tuple[str, str | None]:
     return name, value if sep else None
 
 
-def _strip_index_url_credentials(url: str) -> tuple[str, bool]:
-    """Return ``(clean_url, has_credentials)`` for a pip index URL.
-
-    Strips any ``user:password@`` userinfo so the literal secret can never
-    reach pyproject.toml. A credential-free URL is returned verbatim so it
-    round-trips exactly (e.g. ``==`` against the original holds).
-
-    Works on the raw netloc string rather than :class:`urllib.parse.SplitResult`
-    properties, which raise ``ValueError`` on malformed ports.
-    """
-    parts = urlsplit(url)
-    netloc = parts.netloc
-    if "@" not in netloc:
-        return url, False
-    userinfo, _, hostpart = netloc.rpartition("@")
-    clean_netloc = hostpart
-    clean = urlunsplit(
-        (parts.scheme, clean_netloc, parts.path, parts.query, parts.fragment)
-    )
-    return clean, bool(userinfo.strip())
-
-
-def _uv_index_env_token(name: str) -> str:
-    """uv env-var token for an index name: uppercase, non-alnum -> ``_``.
-
-    ``UV_INDEX_<NAME>_USERNAME`` / ``_PASSWORD`` per
-    https://docs.astral.sh/uv/concepts/indexes/#authentication
-    """
-    return re.sub(r"[^A-Z0-9]", "_", name.upper())
-
-
 def _index_name_from_url(url: str, used_names: set[str]) -> str:
     """Derive a unique, uv-compatible index name from the URL host.
 
@@ -207,23 +173,30 @@ def _index_name_from_url(url: str, used_names: set[str]) -> str:
 
 
 def _build_index_entries(raw_urls: list[str]) -> list[IndexEntry]:
-    """Build de-duplicated, credential-free index entries from raw URLs.
-
-    Duplicate URLs (after credential stripping) collapse to one entry, matching
-    pip's own de-duplication of repeated ``--extra-index-url`` lines.
-    """
+    """De-duplicated index entries from raw pip index URLs."""
     entries: list[IndexEntry] = []
     used_names: set[str] = set()
     seen_urls: set[str] = set()
     for raw in raw_urls:
-        clean, has_credentials = _strip_index_url_credentials(raw)
+        parts = urlsplit(raw)
+        netloc = parts.netloc
+        if "@" in netloc:
+            userinfo, _, host = netloc.rpartition("@")
+            clean = urlunsplit(
+                (parts.scheme, host, parts.path, parts.query, parts.fragment)
+            )
+            has_credentials = bool(userinfo.strip())
+        else:
+            clean = raw
+            has_credentials = False
         if clean in seen_urls:
             continue
         seen_urls.add(clean)
         entries.append(
             IndexEntry(
                 name=_index_name_from_url(clean, used_names),
                 url=clean,
+                raw_url=raw,
                 has_credentials=has_credentials,
             )
         )
@@ -288,10 +261,9 @@ def _parse_requirement_line(
 ) -> tuple[str | None, list[str], list[str]]:
     """Parse one requirements.txt line into ``(dep, index_urls, skipped)``.
 
-    ``dep`` is the dependency with all inline pip-options removed, or ``None``
-    when the line carried only options. ``index_urls`` are raw index URLs
-    (credentials stripped later, collectively). ``skipped`` lists each dropped
-    option by name (may contain duplicates across ``--hash`` repeats).
+    ``dep`` is the dependency with inline pip-options removed, or ``None``
+    when the line carried only options. ``skipped`` may contain duplicates
+    (e.g. repeated ``--hash`` on the same line).
     """
     tokens = line.split()
     dep_tokens: list[str] = []
@@ -544,23 +516,22 @@ def print_migration_info(self) -> None:
                     index.url,
                 )
                 continue
-            token = _uv_index_env_token(index.name)
+            token = re.sub(r"[^A-Z0-9]", "_", index.name.upper())
             log.warning(
-                "migration-index-credentials-stripped: name=%s url=%s "
+                "migration-index-credentials-present: name=%s url=%s "
                 "env_username=UV_INDEX_%s_USERNAME env_password=UV_INDEX_%s_PASSWORD",
                 index.name,
                 index.url,
                 token,
                 token,
             )
             print(
-                f"Warning: index '{index.name}' had embedded credentials in "
-                f"requirements.txt. They are NOT stored in pyproject.toml."
+                f"WARNING: index '{index.name}' has embedded credentials "
+                f"in pyproject.toml — they work now but may leak secrets to VCS."
             )
             print(
-                f"Provide them via environment variables "
-                f"UV_INDEX_{token}_USERNAME and UV_INDEX_{token}_PASSWORD "
-                f"(or a ~/.netrc entry for {index.url})."
+                f"Consider replacing them with environment variables "
+                f"UV_INDEX_{token}_USERNAME and UV_INDEX_{token}_PASSWORD."
             )
             print()
 
@@ -588,14 +559,7 @@ def print_migration_info(self) -> None:
         )
 
     def _parse_requirements_file(self) -> RequirementsTxtInfo:
-        """Parse requirements.txt into dependencies, indexes, and skipped options.
-
-        pip-options (``--index-url``, ``--hash``, ...) are separated from real
-        dependencies so they never leak into ``[project] dependencies`` as
-        invalid PEP 508 specifiers. Index URLs become :class:`IndexEntry`
-        objects (credentials stripped); unsupported options are recorded in
-        ``skipped_options`` for the migrate warning.
-        """
+        """Parse requirements.txt into :class:`RequirementsTxtInfo`."""
         log.debug("parse-requirements-file: path=%s", self.requirements_path)
         content = self.requirements_path.read_text()
         raw_lines = [
@@ -706,7 +670,7 @@ def _generate_content_with_project(
     def _uv_index_section(indexes: list[IndexEntry]) -> str:
         """Render ``[[tool.uv.index]]`` entries from migrated index options."""
         blocks = [
-            f'[[tool.uv.index]]\nname = "{index.name}"\nurl = "{index.url}"'
+            f'[[tool.uv.index]]\nname = "{index.name}"\nurl = "{index.raw_url}"'
             for index in indexes
         ]
         return "\n\n".join(blocks) + "\n"
@@ -1880,14 +1844,17 @@ def migrate(
         pyproject = pyproject.migrate_from_requirements_txt()
         pyproject.print_migration_info()
 
+        # Write .gitignore BEFORE uv lock so the ignore entries survive a
+        # lock failure and users are not suprised by untracked files.
+        ensure_gitignore(self.base, _GITIGNORE_ENTRIES)
+
         uv_lock_out = self._uv_lock(uv, diff=False)
         print(uv_lock_out)
 
         # Also cleans up old .appenv hash-based venvs
         print("Preparing/cleaning .appenv directory ...")
         self._prepare_appenv_dir()
 
-        ensure_gitignore(self.base, _GITIGNORE_ENTRIES)
         log.info("migrate-completed: base=%s", self.base)
         print("\n=== Pyproject Migration completed ===")
         print("requirements.{txt,lock} kept as legacy. You can delete these files now.")

src/appenv.pyi

@@ -30,6 +30,7 @@ class UsageArgumentParser(argparse.ArgumentParser):
 class IndexEntry(NamedTuple):
     name: str
     url: str
+    raw_url: str
     has_credentials: bool
 
 _PIP_INDEX_URL_OPTIONS: Final[frozenset[str]]
@@ -44,8 +45,6 @@ class RequirementsTxtInfo(NamedTuple):
     skipped_options: list[str]
 
 def _split_pip_option_token(token: str) -> tuple[str, str | None]: ...
-def _strip_index_url_credentials(url: str) -> tuple[str, bool]: ...
-def _uv_index_env_token(name: str) -> str: ...
 def _index_name_from_url(url: str, used_names: set[str]) -> str: ...
 def _build_index_entries(raw_urls: list[str]) -> list[IndexEntry]: ...
 def _consume_next_value(

tests/test_gitignore_consistency.py

@@ -9,6 +9,10 @@
 command never silently commits the `.appenv/venv/` tree.
 """
 
+import subprocess
+
+import pytest
+
 import appenv
 
 EXPECTED_GITIGNORE_ENTRIES = appenv._GITIGNORE_ENTRIES
@@ -136,3 +140,37 @@ def capture(base, entries):
     )
     assert init_entries is appenv._GITIGNORE_ENTRIES
     assert init_entries == EXPECTED_GITIGNORE_ENTRIES
+
+
+def test_migrate_writes_gitignore_even_when_uv_lock_fails(
+    tmp_path, monkeypatch, app_env
+):
+    """reorder-migrate::gitignore-before-lock — ensure_gitignore runs BEFORE uv
+    lock, so .gitignore is written even when uv lock crashes.
+
+    Regression test for the original bug: migrate() called ensure_gitignore
+    AFTER _uv_lock, so a lock failure (e.g. missing credentials for a private
+    index) prevented .gitignore from being written.
+    """
+    base = tmp_path
+    (base / "requirements.txt").write_text(
+        "--extra-index-url https://deploy:s3cr3t@gitlab.example.com/simple\nrequests\n"
+    )
+
+    def _raise(self, uv, diff=False):
+        raise subprocess.CalledProcessError(returncode=1, cmd=["uv", "lock"])
+
+    monkeypatch.setattr(appenv.AppEnv, "_uv_lock", _raise)
+
+    env = app_env(base)
+    with pytest.raises(subprocess.CalledProcessError):
+        env.migrate()
+
+    # .gitignore was written despite the crash.
+    gitignore = base / ".gitignore"
+    assert gitignore.exists()
+    gitignore_lines = gitignore.read_text().splitlines()
+    for entry in EXPECTED_GITIGNORE_ENTRIES:
+        assert entry in gitignore_lines, (
+            f"migrate must add {entry!r} to .gitignore even when uv lock fails"
+        )

tests/test_migrate_pip_options.py

@@ -10,20 +10,20 @@
 What is pinned (decided by the spec):
   * pip-options are separated from real dependencies and never written as deps.
   * ``--index-url`` / ``--extra-index-url`` become ``[[tool.uv.index]]`` entries.
-  * embedded credentials are stripped (the literal secret never reaches
-    pyproject.toml or stdout) and the user is warned with the exact
-    ``UV_INDEX_<NAME>_USERNAME`` / ``_PASSWORD`` env var names, where ``<NAME>``
-    is derived from the index name per uv's documented mechanism
-    (https://docs.astral.sh/uv/concepts/indexes/#authentication).
+   * embedded credentials are kept in pyproject.toml (so ``uv lock`` succeeds
+     without manual env-var setup), but the user is warned with the exact
+     ``UV_INDEX_<NAME>_USERNAME`` / ``_PASSWORD`` env var names and told to
+     consider removing the credentials from the URL.
   * options without a pyproject equivalent (``--hash``, ``--no-binary``,
     ``--only-binary``, ``--require-hashes``) are dropped with a warning that
     lists the skipped options.
 
 Deliberately NOT pinned (spec leaves open, so the implementer chooses):
-  * how the index ``name`` is derived from the URL.
-  * the exact stored URL representation when credentials are present (clean URL
-    vs ``$VAR`` placeholder). Only the security guarantee (no literal secret
-    leak) and the env-var guidance are asserted.
+   * how the index ``name`` is derived from the URL.
+   * the exact stored URL representation when credentials are present.
+     The security guarantee (no literal secret leak to stdout) and the
+     env-var guidance are asserted. Credentials in pyproject.toml are an
+     accepted trade-off — the warning tells the user to remove them.
 """
 
 import re
@@ -185,17 +185,20 @@ def test_migrate_assigns_distinct_names_to_multiple_indexes(migrate_reqs):
 
 
 # ============================================================================
-# Credentials: stripped (no literal secret leak) + env-var guidance.
+# Credentials: kept in pyproject (so uv lock works), warned to stdout.
 # ============================================================================
 
 
-def test_migrate_strips_index_credentials_and_warns_env_vars(migrate_reqs, patterns):
-    """Credentialed index URL: the secret never reaches pyproject or stdout, and
-    the user is told the exact ``UV_INDEX_<NAME>_USERNAME`` / ``_PASSWORD`` names.
+def test_migrate_keeps_credentials_in_pyproject_and_warns_env_vars(
+    migrate_reqs, patterns
+):
+    """Credentialed index URL: the secret IS kept in pyproject.toml (so ``uv
+    lock`` works without manual env-var setup), but the user is warned with
+    the exact ``UV_INDEX_<NAME>_USERNAME`` / ``_PASSWORD`` names and instructed
+    to consider removing credentials from the URL.
 
-    Merges the security contract (no literal leak anywhere) with the UX contract
-    (env-var guidance emitted). The exact derived token is still asserted per
-    index so a token-derivation regression cannot hide behind a wildcard.
+    Security contract: no literal secret leak to stdout (pyproject.toml is the
+    accepted trade-off — the warning tells the user to fix it).
     """
     result = migrate_reqs(
         "--extra-index-url https://deploy:s3cr3t@gitlab.example.com/simple\nrequests\n"
@@ -205,8 +208,8 @@ def test_migrate_strips_index_credentials_and_warns_env_vars(migrate_reqs, patte
     assert len(indexes) == 1
     assert "gitlab.example.com" in indexes[0]["url"]
 
-    # Security: the literal secret must never appear in pyproject.
-    assert "s3cr3t" not in result.pyproject
+    # Secret is in pyproject.toml so uv lock can authenticate.
+    assert "s3cr3t" in result.pyproject
 
     # UX: the exact derived env-var token is advertised for each credentialed index.
     for idx in indexes:
@@ -308,7 +311,11 @@ def test_migrate_warns_lists_all_skipped_pip_options(migrate_reqs):
 
 
 def test_migrate_gitlab_private_registry_end_to_end(migrate_reqs):
-    """A GitLab Package Registry requirements.txt migrates cleanly."""
+    """A GitLab Package Registry requirements.txt migrates cleanly.
+
+    Credentials are kept in pyproject (so uv lock works) but not in stdout;
+    the user is warned to replace them with env vars.
+    """
     result = migrate_reqs(
         "--extra-index-url https://deploy:s3cr3t@gitlab.example.com/api/v4/"
         "projects/42/packages/pypi/simple\n"
@@ -323,14 +330,14 @@ def test_migrate_gitlab_private_registry_end_to_end(migrate_reqs):
         "requests",
     ]
 
-    # Index entry created, host preserved, secret stripped everywhere.
+    # Index entry created, host preserved, secret kept in pyproject (so uv
+    # lock works), not leaked to stdout.
     indexes = _extract_uv_indexes(result.pyproject)
     assert len(indexes) == 1
     assert (
         "gitlab.example.com/api/v4/projects/42/packages/pypi/simple"
         in indexes[0]["url"]
     )
-    assert "s3cr3t" not in result.pyproject
     assert "s3cr3t" not in result.stdout
 
     # Credential env-var guidance + skipped-option warnings are emitted.
@@ -405,9 +412,9 @@ def test_migrate_assigns_numeric_suffix_when_index_hosts_collide(migrate_reqs):
 def test_migrate_dedups_repeated_identical_index_url(migrate_reqs):
     """Two identical --extra-index-url lines collapse to a single index entry.
 
-    Drives the de-duplication ``continue`` branch: a URL already seen (after
-    credential stripping) is not emitted a second time, matching pip's own
-    de-duplication of repeated ``--extra-index-url`` lines.
+    Drives the de-duplication ``continue`` branch: a URL already seen (the
+    dedup key is the credential-free ``url``) is not emitted a second time,
+    matching pip's own de-duplication of repeated ``--extra-index-url`` lines.
     """
     result = migrate_reqs(
         "--extra-index-url https://gitlab.example.com/simple\n"