update_command_docs.py

#!/usr/bin/env python3
"""Regenerate the command reference pages under docs/docs/commands/.

The Commands section is a pure reference: every page is derived from what
`datacontract <command> --help` prints, so it can never drift from the CLI.
Groups (`import`, `export`, `dbt`) get a folder with one sub-page per
subcommand. Prose guides live elsewhere (docs/docs/imports, exports, testing);
nothing outside docs/docs/commands/ is touched.

    python update_command_docs.py          # rewrite the pages
    python update_command_docs.py --check  # fail if anything is out of date

Re-run whenever a command, option, or help string changes.
"""

from __future__ import annotations

import json
import re
import shutil
import sys
from pathlib import Path
from typing import Any

import typer.main

from datacontract.cli import app

# typer >= 0.27 builds its command tree from a vendored click fork, so the objects
# below are not instances of the installed `click` classes. Everything here is
# duck-typed against the click API instead of isinstance-checked.

DOCS = Path("docs/docs/commands")

# Sidebar order of the top-level commands: workflow order, not alphabetical.
COMMAND_ORDER = [
    "init",
    "edit",
    "lint",
    "changelog",
    "test",
    "dbt",
    "ci",
    "export",
    "import",
    "catalog",
    "publish",
    "api",
]

# The prose guide each top-level command belongs to. The reference page states
# what the options are; the guide states what to do with them, so every page
# links to its counterpart. Paths are relative to docs/docs/commands/.
GUIDES = {
    "test": ("Test your Data", "../testing/index.md"),
    "dbt": ("Sync with dbt", "../dbt.md"),
    "edit": ("Edit your Contract", "../editor.md"),
    "ci": ("Scheduling", "../scheduling/index.md"),
    "api": ("Run as a web server", "../api.md"),
    "publish": ("Publish to Entropy Data", "../entropy-data.md"),
    "lint": ("Open Data Contract Standard", "../open-data-contract-standard.md"),
    "import": ("Imports", "../imports/index.md"),
    "export": ("Exports", "../exports/index.md"),
}

# Groups whose subcommands have a one-page guide of their own, as
# `<command>` -> `<docs/docs folder>`. The page only exists for some formats,
# so the link is emitted per subcommand and only when the guide is there.
SUBCOMMAND_GUIDES = {"import": "imports", "export": "exports"}

BANNER = "{/* AUTOGENERATED from `datacontract --help`: do not edit by hand; regenerate with update_command_docs.py */}"


def escape(text: str | None) -> str:
    """Make help text safe for an MDX table cell.

    MDX reads `{...}` as a JSX expression and `<...>` as a tag, both of which
    fail the docs build, so they are escaped. Backslashes are left alone: the
    help strings already escape for the terminal renderer (`\\[`), and doubling
    them would print the backslash instead of the bracket.
    """
    if not text:
        return "—"
    out = " ".join(text.split())
    for char in ("|", "<", ">", "{", "}"):
        out = out.replace(char, f"\\{char}")
    return out


def is_group(cmd: Any) -> bool:
    """A command group is a command that has subcommands."""
    return hasattr(cmd, "commands")


def format_default(param: Any) -> str:
    default = param.default
    if param.required:
        return "required"
    if default is None or default == "":
        return "—"
    if isinstance(default, bool):
        # Boolean flags read better as the flag that is on by default.
        if param.secondary_opts:
            return f"`{param.opts[0] if default else param.secondary_opts[0]}`"
        return "on" if default else "off"
    if isinstance(default, (list, tuple)):
        return "—" if not default else f"`{', '.join(str(d) for d in default)}`"
    value = getattr(default, "value", default)
    return f"`{value}`"


def metavar(param: Any) -> str:
    """`[LOCATION]`, `V1`, `[CONTRACT]...` — how an argument appears in the usage line.

    Spelled out here rather than taken from `make_metavar()`, whose signature and
    output both changed across click 8.2 and typer 0.27; the pages must render the
    same whichever version is installed.
    """
    name = param.metavar or param.name.upper()
    if not param.required:
        name = f"[{name}]"
    return name if param.nargs == 1 else f"{name}..."


def usage(cmd: Any, path: str) -> str:
    pieces = [cmd.options_metavar] if cmd.options_metavar else []
    pieces += [metavar(p) for p in cmd.params if p.param_type_name == "argument"]
    if is_group(cmd):
        pieces.append(cmd.subcommand_metavar)
    return f"datacontract {path} " + " ".join(pieces)


def arguments_table(cmd: Any) -> list[str]:
    args = [p for p in cmd.params if p.param_type_name == "argument"]
    if not args:
        return []
    rows = ["| Argument | Default | Description |", "|---|---|---|"]
    for p in args:
        rows.append(f"| `{metavar(p)}` | {format_default(p)} | {escape(getattr(p, 'help', None))} |")
    return rows + [""]


def options_table(cmd: Any) -> list[str]:
    opts = [p for p in cmd.params if p.param_type_name == "option" and not _is_help(p)]
    if not opts:
        return []
    rows = ["| Option | Default | Description |", "|---|---|---|"]
    for p in opts:
        names = " / ".join(f"`{o}`" for o in list(p.opts) + list(p.secondary_opts))
        rows.append(f"| {names} | {format_default(p)} | {escape(p.help)} |")
    return rows + [""]


def _is_help(param: Any) -> bool:
    return "--help" in param.opts


def first_sentence(text: str) -> str:
    """The frontmatter description is a one-line summary, not the full help."""
    for end in (". ", ".\n"):
        if end in text:
            return text.split(end)[0] + "."
    return text


def frontmatter(title: str, description: str, position: int, slug: str | None = None) -> list[str]:
    safe = first_sentence(description).replace('"', "'")
    lines = ["---", f"sidebar_position: {position}", f'title: "{title}"']
    if slug:
        lines.append(f"slug: {slug}")
    return lines + [f'description: "{safe}"', "---", ""]


def short_help(cmd: Any) -> str:
    text = (cmd.help or cmd.short_help or "").strip()
    return " ".join(text.split()) if text else "No description available."


def summary(cmd: Any) -> str:
    """A one-sentence description for a table cell.

    `edit` and `api` document their whole behaviour in the help string; the full
    text belongs on their own page, not in a row of the command table.
    """
    return escape(first_sentence(short_help(cmd)))


def guide_link(command: str, depth: int) -> list[str]:
    """A "see the guide" line for a command, or nothing when it has no guide.

    `depth` is how many folders below docs/docs/commands/ the page sits, so the
    relative link stays correct for both `commands/test.md` and
    `commands/import/postgres.md`.
    """
    guide = GUIDES.get(command)
    if guide is None:
        return []
    label, target = guide
    return [f"Guide: **[{label}]({'../' * depth}{target})**.", ""]


def subcommand_guide_link(group_name: str, sub_name: str) -> list[str]:
    """The same, for the per-format guide of an `import`/`export` subcommand.

    Not every subcommand has one — `export dbt` is a removed command, and the
    `import unity` guide is an unlisted stub — so the page has to exist and be
    listed. The link text is the guide's own title, so it reads the same here as
    it does in the sidebar.
    """
    folder = SUBCOMMAND_GUIDES.get(group_name)
    if folder is None:
        return []
    guide = DOCS.parent / folder / f"{sub_name}.md"
    if not guide.exists():
        return []
    text = guide.read_text()
    if "unlisted: true" in text:
        return []
    title = re.search(r'^title: "(.*)"$', text, re.M).group(1)
    return [f"Guide: **[{title}](../../{folder}/{sub_name}.md)**.", ""]


def render_command(cmd: Any, path: str, position: int, guide: list[str] | None = None) -> str:
    """A reference page for one leaf command."""
    lines = frontmatter(path, short_help(cmd), position)
    lines += [f"# `datacontract {path}`", "", BANNER, "", short_help(cmd), ""]
    lines += ["```bash", usage(cmd, path), "```", ""]
    lines += arguments_table(cmd)
    lines += options_table(cmd)
    if cmd.epilog:
        example = cmd.epilog.strip()
        example = example[len("Example:") :].strip() if example.startswith("Example:") else example
        lines += ["```bash", example, "```", ""]
    lines += guide if guide is not None else guide_link(path, depth=0)
    return "\n".join(lines).rstrip() + "\n"


def render_group_index(group: Any, name: str, position: int) -> str:
    """The landing page for a command group, listing its subcommands."""
    lines = frontmatter(name, short_help(group), position)
    lines += [f"# `datacontract {name}`", "", BANNER, "", short_help(group), ""]
    lines += ["```bash", usage(group, name), "```", ""]
    lines += options_table(group)
    lines += ["| Subcommand | Description |", "|---|---|"]
    for sub_name in sorted(group.commands):
        sub = group.commands[sub_name]
        lines.append(f"| [`{sub_name}`](./{sub_name}.md) | {summary(sub)} |")
    lines.append("")
    lines += guide_link(name, depth=1)
    return "\n".join(lines).rstrip() + "\n"


def render_root_index(root: Any, position: int = 0) -> str:
    """The Commands landing page: the global options and the command table.

    Generated like every other page here, so `datacontract --help` stays the
    single source of truth for the global options too — they appear on no other
    page.
    """
    lines = frontmatter("Commands", "Reference for every Data Contract CLI command.", position, slug="/commands")
    lines += [
        "# Commands",
        "",
        BANNER,
        "",
        "The `datacontract` CLI groups its functionality into the commands below. Run "
        "`datacontract --help` or `datacontract <command> --help` at any time to see the "
        "same information in your terminal.",
        "",
        "```bash",
        usage(root, "").replace("datacontract  ", "datacontract "),
        "```",
        "",
    ]

    lines += ["## Global options", ""]
    lines += options_table(root)
    lines += ["Most commands additionally accept `--debug` for verbose logging.", ""]

    lines += ["## Commands", "", "| Command | Description |", "|---|---|"]
    for name in sorted(root.commands, key=lambda n: COMMAND_ORDER.index(n) if n in COMMAND_ORDER else 99):
        cmd = root.commands[name]
        target = f"./{name}/index.md" if is_group(cmd) else f"./{name}.md"
        lines.append(f"| [`{name}`]({target}) | {summary(cmd)} |")
    lines.append("")

    lines += ["## Common usage", "", "```bash"]
    for name in sorted(root.commands, key=lambda n: COMMAND_ORDER.index(n) if n in COMMAND_ORDER else 99):
        epilog = (root.commands[name].epilog or "").strip()
        if epilog.startswith("Example:"):
            lines.append(epilog[len("Example:") :].strip().splitlines()[0].strip())
    lines += ["```", ""]
    return "\n".join(lines).rstrip() + "\n"


def write(path: Path, content: str, changed: list[str]) -> None:
    if path.exists() and path.read_text() == content:
        return
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(content)
    changed.append(str(path))


def generate(changed: list[str]) -> set[Path]:
    """Write every page; return the set of files that should exist."""
    root = typer.main.get_command(app)
    index = DOCS / "index.md"
    write(index, render_root_index(root), changed)
    # _category_.json holds the position of the whole Commands section within
    # the docs sidebar, which the CLI knows nothing about: hand-written.
    expected: set[Path] = {index, DOCS / "_category_.json"}

    for name, cmd in root.commands.items():
        position = COMMAND_ORDER.index(name) + 1 if name in COMMAND_ORDER else 99
        if is_group(cmd):
            folder = DOCS / name
            index = folder / "index.md"
            write(index, render_group_index(cmd, name, position), changed)
            expected.add(index)
            category = folder / "_category_.json"
            write(
                category,
                json.dumps(
                    {"label": name, "position": position, "link": {"type": "doc", "id": f"commands/{name}/index"}},
                    indent=2,
                )
                + "\n",
                changed,
            )
            expected.add(category)
            for i, sub_name in enumerate(sorted(cmd.commands), start=1):
                page = folder / f"{sub_name}.md"
                guide = subcommand_guide_link(name, sub_name)
                write(page, render_command(cmd.commands[sub_name], f"{name} {sub_name}", i, guide), changed)
                expected.add(page)
        else:
            page = DOCS / f"{name}.md"
            write(page, render_command(cmd, name, position), changed)
            expected.add(page)
    return expected


def main() -> int:
    check = "--check" in sys.argv
    if check:
        backup = Path(str(DOCS) + ".bak")
        shutil.copytree(DOCS, backup, dirs_exist_ok=True)

    changed: list[str] = []
    expected = generate(changed)
    stale = [p for p in DOCS.rglob("*") if p.is_file() and p not in expected]
    for path in stale:
        path.unlink()
        changed.append(f"{path} (removed)")

    if check:
        shutil.rmtree(DOCS)
        shutil.move(str(Path(str(DOCS) + ".bak")), str(DOCS))
        if changed:
            print("Command docs are out of date. Run: python update_command_docs.py")
            for c in sorted(changed):
                print(f"  {c}")
            return 1
        print("Command docs are up to date.")
        return 0

    for c in sorted(changed):
        print(f"  {c}")
    print(f"{len(changed)} file(s) written, {len(expected)} page(s) total.")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())