diff --git a/.github/workflows/fern-docs-ci.yml b/.github/workflows/fern-docs-ci.yml index 7ed973be9f..d942e9a9b3 100644 --- a/.github/workflows/fern-docs-ci.yml +++ b/.github/workflows/fern-docs-ci.yml @@ -36,6 +36,10 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # check-doc-version-sync resolves stack release tags to commits. + fetch-depth: 0 + fetch-tags: true - name: Setup Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 @@ -63,3 +67,10 @@ jobs: - name: Fern check run: fern check + + - uses: actions/setup-go@v5 + with: + go-version-file: tools/go-toolchain/go.mod + + - name: Check generated documentation blocks match the version catalog + run: ./tools/ci/check-doc-version-sync diff --git a/.markdownlintignore b/.markdownlintignore index ef9944932d..f63b9468e1 100644 --- a/.markdownlintignore +++ b/.markdownlintignore @@ -7,13 +7,15 @@ vendor/ third_party/ # Frozen versioned documentation. docs/AGENTS.md states that docs/v*/ and -# docs/cp-*/ must not be edited without an explicit request for a historical -# docs fix, so linting these trees can only produce findings that must never be -# acted on. +# docs/-/ must not be edited without an explicit request for a +# historical docs fix, so linting these trees can only produce findings that +# must never be acted on. # -# The [0-9] keeps docs/version-catalog/, which is not frozen, in scope. +# The [0-9] keeps docs/version-catalog/, which is not frozen, in scope. The +# per-stack pattern has no trailing slash on purpose: markdownlint-cli 0.49 +# does not match docs/-/ files against "docs/*-[0-9]*/". docs/v[0-9]*/ -docs/cp-*/ +docs/*-[0-9]* # Vendored Helm subcharts. Same rationale as vendor/: upstream chart content # this repository redistributes rather than authors. @@ -34,13 +36,13 @@ CLAUDE.md # the heading and MD041 can never pass. An inline disable is not an option # either: docs/ is rendered as MDX, where an HTML comment is a parse error, and # MDX's own {/* */} comment is not a markdownlint directive. MD041 is their only -# violation. docs/user/ copies are symlinks to the docs/dev/ originals. +# violation. docs/self-managed/ copies are symlinks to the docs/dev/ originals. docs/dev/grpc-load-test-sli-guide.md docs/dev/http-load-test-sli-guide.md docs/dev/http-load-testing.md -docs/user/grpc-load-test-sli-guide.md -docs/user/http-load-test-sli-guide.md -docs/user/http-load-testing.md +docs/overview/grpc-load-test-sli-guide.md +docs/overview/http-load-test-sli-guide.md +docs/overview/http-load-testing.md # Generated file. tools/collect-dependencies writes dependencies.md and CI # fails if it differs from the generator's output, so it cannot be edited to diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e6fd84d7b3..1a91bdde7f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -165,19 +165,21 @@ For more information, see the [Developer Certificate of Origin](https://develope ## Documentation Contributions -Documentation content lives under `docs/`. Fern publishes that content using version-specific navigation files under `fern/versions/`, and `fern/docs.yml` declares the public version list. +Documentation content lives under `docs/`. Fern publishes one site with four products: Overview plus one product per Helm stack. Each stack product has its own version menu. `fern/docs.yml` declares the products and their versions, and navigation files live under `fern/products/`. | Path | Audience | Published | Versioning role | |---|---|---|---| -| `docs/user/` | Customers | Yes | Source content for the default `main` docs version. `fern/versions/main.yml` points here and publishes at `/nvcf/`. | -| `docs/v0.5/` | Customers using release 0.5 | Yes | Frozen content for the `0.5` docs version. `fern/versions/v0.5.yml` points here and publishes at `/nvcf/v0.5/`. | -| `docs/dev/` | Contributors / internal dev | Only if symlinked | Developer-oriented source pages. These are published only when a symlink from a versioned content tree, usually `docs/user/`, is listed in that version's Fern nav. | -| `fern/docs.yml` | Docs site | Yes | Declares public versions, display names, slugs, and the nav file for each version. | -| `fern/versions/*.yml` | Docs site | Yes | Defines navigation and page order for one published version. Page paths are relative to the version file. | +| `docs/overview/` | Customers | Yes | Unversioned shared content: compatibility matrix, quickstart, manifest, image mirroring, local development, shared images and samples. Publishes at `/nvcf/overview/`. | +| `docs/self-managed/`, `docs/compute-plane/`, `docs/observability/` | Customers | Yes | Top-of-tree content for one stack, published as that product's `dev` version at `/nvcf//dev/`. | +| `docs/-/` | Customers on a release train | Yes | Frozen content for one stack train, for example `docs/observability-1.3/`. Publishes at `/nvcf///`. | +| `docs/v0.5/`, `docs/v0.6.0/`, `docs/v0.6.1/`, `docs/cp-*/` | Customers on legacy releases | Yes | Frozen legacy full-tree content from before the per-stack split, attached to the Self-Managed Stack product. | +| `docs/dev/` | Contributors / internal dev | Only if symlinked | Developer-oriented source pages. These are published only when a symlink from a product tree is listed in that product's Fern nav. | +| `fern/docs.yml` | Docs site | Yes | Declares products, their versions, display names, slugs, and the nav file for each version. | +| `fern/products/overview.yml`, `fern/products//*.yml` | Docs site | Yes | Defines navigation and page order for one product version. Page paths are relative to the nav file. | -Use `docs/user/` for changes that should appear in the default `main` docs. Update `docs/v0.5/` only for fixes that must also apply to the 0.5 release docs. When adding, renaming, moving, or removing a published page, update the matching `fern/versions/.yml` file. +A page belongs to exactly one product. Use the product tree that owns the page for changes that should appear in `dev`. Edit frozen trees only for fixes that must also apply to that released version. When adding, renaming, moving, or removing a published page, update the matching nav file under `fern/products/`. Links to pages in another product must be absolute site paths such as `/nvcf/overview/quickstart`, because Fern resolves relative links inside the rendering product. -All navigation sections use `skip-slug: true`, so each page title becomes a flat URL slug within its published version. Keep page titles unique and descriptive within the version nav. Run `fern check` to validate the docs after any navigation or link change. Preview locally with `fern docs dev` from the `fern/` directory. +All navigation sections use `skip-slug: true`, so each page title becomes a flat URL slug within its product version. Keep page titles unique and descriptive within each product nav. Run `fern check` to validate the docs after any navigation or link change. Preview locally with `fern docs dev` from the `fern/` directory. --- diff --git a/README.md b/README.md index 369c0cfcf7..cefc1d53d1 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -![NVCF banner](docs/user/images/nvcf-banner.svg) +![NVCF banner](docs/overview/images/nvcf-banner.svg) [![bazel](https://github.com/NVIDIA/nvcf/actions/workflows/bazel.yml/badge.svg?branch=main)](https://github.com/NVIDIA/nvcf/actions/workflows/bazel.yml?query=branch%3Amain) -[Docs](https://docs.nvidia.com/nvcf/overview) | [Roadmap](#roadmap) | [Installation](docs/user/installation.md) | [API Reference](docs/user/api.md) | [Contributing](CONTRIBUTING.md) | [License](#license) | [build.nvidia.com Powered By NVCF](https://build.nvidia.com/) +[Docs](https://docs.nvidia.com/nvcf/overview) | [Roadmap](#roadmap) | [Installation](docs/self-managed/installation.md) | [API Reference](docs/overview/api.md) | [Contributing](CONTRIBUTING.md) | [License](#license) | [build.nvidia.com Powered By NVCF](https://build.nvidia.com/) > **Repository move in progress.** This repo is transferring from `NVIDIA/nvcf` to > `dsx-ai-factory/nvcf`. Now targeted for **after 2026-09-27** — exact date and @@ -23,7 +23,7 @@ examples, CLI code, agent skills, and validation tooling. ## Architecture -![NVCF architecture](docs/user/images/nvcf-high-level-stack.svg) +![NVCF architecture](docs/overview/images/nvcf-high-level-stack.svg) NVCF runs as Kubernetes services that manage function lifecycle, invocation routing, GPU cluster integration, artifact access, secrets, observability, and @@ -44,7 +44,7 @@ At a high level: The following diagram shows how self-managed NVCF can span regions and GPU clusters. -NVCF multi-region and multi-cluster architecture +NVCF multi-region and multi-cluster architecture ### Workload types @@ -90,8 +90,8 @@ nvcf-cli function invoke --request-body '{"message": "hello world"}' ``` For the full setup, cleanup, and configuration flow, see -[`docs/user/cli.md`](docs/user/cli.md) and -[`docs/user/quickstart.md`](docs/user/quickstart.md). +[`docs/overview/cli.md`](docs/overview/cli.md) and +[`docs/overview/quickstart.md`](docs/overview/quickstart.md). ## Repository map @@ -102,7 +102,7 @@ For the full setup, cleanup, and configuration flow, see | Compute plane | [`src/compute-plane-services/`](src/compute-plane-services/) | GPU cluster integration, cache services, image credentials, ESS Agent, and telemetry collection. | | CLI and libraries | [`src/clis/`](src/clis/), [`src/libraries/`](src/libraries/) | User and developer clients plus shared Go and Python code. | | Deployment | [`deploy/`](deploy/), [`migrations/`](migrations/) | Helm charts, stack installation, infrastructure services, and datastore migrations. | -| Documentation | [`docs/user/`](docs/user/index.md), [`docs/dev/`](docs/dev/), [`fern/`](fern/) | Self-managed user docs, developer docs, and published docs navigation. | +| Documentation | [`docs/overview/`](docs/overview/index.md), [`docs/self-managed/`](docs/self-managed/), [`docs/compute-plane/`](docs/compute-plane/), [`docs/observability/`](docs/observability/), [`docs/dev/`](docs/dev/), [`fern/`](fern/) | Per-stack user docs, shared overview docs, developer docs, and published docs navigation. | | Examples | [`examples/`](examples/) | Local development guides, function samples, and load-test assets. | | Tools | [`tools/`](tools/) | Build, docs, dependency, license, and validation utilities. | | AI tooling | [`ai-tooling/`](ai-tooling/) | Public agent skills and workflow helpers for NVCF users and developers. | diff --git a/RELEASE.md b/RELEASE.md index 42b8de9994..c28f6836a7 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -61,6 +61,42 @@ releases no stack version at all. internal publish lanes accept `-rc.N`, so the stack is packaged and published without a version landing on its stable line. +The three stacks were level-set to `1.0.0` when this model started. From that +point each stack advances on its own trains and its own schedule. A +compute-plane `1.2` says nothing about which self-managed or observability +train it runs with; the compatibility matrix does, see "Stack compatibility and +upgrade stops" below. + +When a tag is pushed, the `tag` job resolves the publishing stack from the tag +alone, using the `tag_format` in `tools/ci/github-release-subprojects.json`, +and attaches that stack's `resolved_inventory_asset` to the GitHub Release. +Which branch cut the tag does not matter, so release branching did not change +how inventories reach a release. + +### Stack compatibility and upgrade stops + +- Only trains N and N-1 of a stack are maintained. A train older than N-1 + receives no further patches. +- Patch upgrades within a train are always supported. +- Upgrading across more than one train is done one train at a time: move to + the latest patch of each intermediate train before moving to the next. Each + train is an upgrade stop. +- The compatibility matrix at `docs/overview/compatibility-matrix.md` is the + source of truth for which trains of the three stacks run together. It is + generated by `tools/docs-version-sync` from the `compatibility:` block in + `docs/version-catalog/main.yaml`. A stack change that alters what it can run + with must update that block in the same change. + +### Stack documentation + +Each stack is published as its own Fern product with its own version list, so +documentation for one stack can be frozen without touching the other two. +After a train's first release, freeze that stack's docs with: + +```sh +./tools/scripts/cut-docs-version.sh --stack --train X.Y +``` + `nvca` used the same model until the 3.3 line and now releases from `main` like every other subproject. Nothing publishes a `-dev.N` any more, and none of the ones published under the older form of this model are still in the @@ -153,7 +189,9 @@ for, or must not carry, the current state of `main`. Support window: where a subproject does maintain release branches, only the latest minor release train and the one before it (N and N-1) are maintained. A release branch older than N-1 is effectively end of life and does not -receive further backports. +receive further backports. For the `deploy/stacks/` subprojects the window is +counted per stack, and each train is also an upgrade stop; see "Stack +compatibility and upgrade stops". Mechanism: cherry-pick the commit from `main` onto the `release-*` branch, following the same commit and review conventions as `main`. There is no diff --git a/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md b/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md index 0bd38164d1..d9e9b9957a 100644 --- a/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md +++ b/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md @@ -99,5 +99,5 @@ After exploring, suggest the next skill when applicable: - `nvcf-self-managed-installation` for installing, upgrading, or tearing down the stack - `docs/dev/local-development.md` for k3d / local cluster work - `nvcf-self-managed-cli` for `nvcf-cli` usage against an installed stack -- `docs/AGENTS.md` and `fern/versions/dev.yml` for routing the user to a published docs page +- `docs/AGENTS.md`, `fern/products/overview.yml` (Overview), and `fern/products//dev.yml` (stack products) for routing the user to a published docs page - `tools/ci/check-doc-version-sync` for keeping the documentation manifest in sync with the docs version catalog diff --git a/ai-tooling/user/skills/nvcf-self-managed-installation/references/helmfile-structure.md b/ai-tooling/user/skills/nvcf-self-managed-installation/references/helmfile-structure.md index 7a976940a5..5fafd18c48 100644 --- a/ai-tooling/user/skills/nvcf-self-managed-installation/references/helmfile-structure.md +++ b/ai-tooling/user/skills/nvcf-self-managed-installation/references/helmfile-structure.md @@ -206,7 +206,7 @@ must not be used in production. The request router uses `power-of-two` when no load-balancer configuration is set. Configure other routing methods with the -[LLM Request Router Load Balancing](https://github.com/NVIDIA/nvcf/blob/main/docs/user/llm-request-router-load-balancing.md) +[LLM Request Router Load Balancing](https://github.com/NVIDIA/nvcf/blob/main/docs/self-managed/llm-request-router-load-balancing.md) guide. If the sidecar image is mirrored outside the stack's default image registry and diff --git a/deploy/helm/llm-request-router/README.md b/deploy/helm/llm-request-router/README.md index 0f232a097c..5bf5be2e59 100644 --- a/deploy/helm/llm-request-router/README.md +++ b/deploy/helm/llm-request-router/README.md @@ -204,7 +204,7 @@ The chart can pass a Stargate load-balancer config in either of two ways: See the [Stargate load balancer configuration](../../../src/libraries/rust/stargate/docs/load-balancer-configuration.md) for the JSON schema, algorithm behavior, and tuning fields. See -[LLM Request Router Load Balancing](../../../docs/user/llm-request-router-load-balancing.md) +[LLM Request Router Load Balancing](../../../docs/self-managed/llm-request-router-load-balancing.md) for stack ownership, trusted headers, rollout checks, and troubleshooting. ## Local Render diff --git a/deploy/stacks/AGENTS.md b/deploy/stacks/AGENTS.md index e008165846..a8217075c3 100644 --- a/deploy/stacks/AGENTS.md +++ b/deploy/stacks/AGENTS.md @@ -13,6 +13,11 @@ resource. - `nvcf-compute-plane/` owns the NVCF compute-plane stack. - `observability/` owns shared observability infrastructure. - Each stack owns its own `release-inventory.yaml` and release asset. +- Each stack releases from its own `release-deploy/stacks//vX.Y` + branch and advances its trains independently. All three started at `1.0.0`. +- Each stack has its own documentation tree (`docs/self-managed/`, + `docs/compute-plane/`, `docs/observability/`) published as a Fern product + with its own version list. Shared pages live in `docs/overview/`. - Do not reference another stack's Helmfile state from an inventory config. - Keep a dependency in the stack that installs or creates it. - Follow the nearest nested `AGENTS.md` when it adds stack-specific guidance. @@ -27,9 +32,14 @@ For every dependency change: 4. Add registry and repository overrides for images that customers must mirror. 5. Record images that do not appear in rendered Kubernetes `image` fields. 6. Update the artifact classification in `docs/version-catalog/main.yaml`. -7. Run the stack tests and the inventory and documentation checks described in +7. If the change alters which trains of another stack this stack works with, + whether the minimum is raised, lowered, or a train entry changes, update + the `compatibility:` block in `docs/version-catalog/main.yaml` in the same + change (for example `compute-plane: "1.2+"`). It generates + `docs/overview/compatibility-matrix.md`. +8. Run the stack tests and the inventory and documentation checks described in [`INVENTORY.md`](INVENTORY.md). -8. After the stack release publishes its inventory asset, update the catalog +9. After the stack release publishes its inventory asset, update the catalog and generated manifest in a documentation sync change. A dependency is not fully distributed when the released inventory or generated @@ -60,7 +70,9 @@ go run -C tools/docs-version-sync . --target main git diff --check ``` -Do not hand-edit generated blocks in `docs/user/manifest.md`. The CI check +Do not hand-edit generated blocks in `docs/overview/manifest.md`, +`docs/overview/compatibility-matrix.md`, or any other generated block under +the product docs trees. The CI check against the latest released inventory is warn-only for now. Generated-document consistency remains blocking. Treat a release-drift warning as follow-up work and keep the local checks clean for a dependency change. diff --git a/deploy/stacks/INVENTORY.md b/deploy/stacks/INVENTORY.md index e3f8eb6e0c..9b9df56566 100644 --- a/deploy/stacks/INVENTORY.md +++ b/deploy/stacks/INVENTORY.md @@ -9,8 +9,9 @@ to pull. Release automation publishes those inventories as separate assets, then the documentation sync combines them into one customer-facing manifest. This split keeps ownership close to the stack that installs the dependency. -Each stack is packaged, tagged, and inventoried independently. Customer support -and qualification apply to one three-stack release set after joint QA. +Each stack is packaged, tagged, inventoried, and documented independently, on +its own release trains. Which trains run together is recorded in the +compatibility matrix, not implied by a shared version. ## Architecture @@ -21,14 +22,18 @@ The flow has three layers: under that stack directory. 1. The release workflow checks out an immutable stack tag, renders the profiles named by that stack, and publishes one resolved JSON inventory with the - matching GitHub Release. + matching GitHub Release. The publishing stack is resolved from the tag + alone, so a tag cut from a `release-deploy/stacks//vX.Y` branch + attaches its inventory the same way. 1. `tools/docs-version-sync` downloads all three released inventories. It validates their plane boundaries, merges compatible artifacts, updates `docs/version-catalog/main.yaml`, and generates the inventory blocks in - `docs/user/manifest.md`. -1. The catalog records the exact control-plane, compute-plane, and - observability versions. A development release set follows the newest stack - releases. A qualified release set uses three versions selected by QA. + `docs/overview/manifest.md` and the compatibility matrix in + `docs/overview/compatibility-matrix.md`. +1. The catalog records the latest released control-plane, compute-plane, and + observability versions, and the `compatibility:` block records which trains + run together. Each stack's documentation is frozen on its own when its + train first releases. The version catalog is the handoff between release facts and public documentation. Released inventories provide immutable versions and source @@ -72,12 +77,10 @@ sequenceDiagram Sync->>Catalog: Open reviewable development docs update Catalog->>Manifest: Generate ownership and optionality - QA->>QA: Qualify one exact three-stack set - QA->>Sync: Approve exact versions and docs version - Sync->>Assets: Download the three selected inventories - Sync->>Catalog: Record qualified release_set - Catalog->>Stable: Snapshot docs and catalog - Stable->>Stable: Make qualified docs the default + QA->>QA: Confirm the minimum train of each other stack this train works with + QA->>Catalog: Update the compatibility block + Catalog->>Stable: Freeze that stack's docs and catalog for the train + Stable->>Stable: Make the train the stack's default docs version ``` ## Ownership Model @@ -89,11 +92,12 @@ Keep each fact in one source: configuration that cannot be derived from an ordinary render. - The resolved JSON inventory records the artifacts found at an immutable stack tag. Release automation publishes it with the stack release. -- The catalog `release_set` records the exact three stack releases represented - by the generated documentation. +- The catalog records the latest released version of each stack and, in the + `compatibility:` block, which trains run together. - `docs/version-catalog/main.yaml` records public distribution locations and human-authored descriptions and source links. -- Generated blocks in `docs/user/manifest.md` present the catalog to users. +- Generated blocks in `docs/overview/manifest.md` and + `docs/overview/compatibility-matrix.md` present the catalog to users. Do not maintain a second hand-written list of versions in the documentation. @@ -213,9 +217,9 @@ go test -C tools/docs-version-sync ./... ./tools/ci/check-docs ``` -By default the update selects the latest stable release of each stack and marks -the release set as `development`. The update is reviewable and does not imply -that QA qualified the selected combination. +By default the update selects the latest stable release of each stack. The +update is reviewable and does not by itself change the `compatibility:` block, +which is maintained by hand when qualification results change. The catalog update retains an exact publication only when its artifact name, type, and version still match. Leave an artifact in `publication_pending` until @@ -227,27 +231,23 @@ release-drift warning does not block a merge. Generated-document consistency remains blocking, and local validation should still return success before a dependency change is complete. -## Qualified Documentation Promotion +## Per-Stack Documentation Freeze -After QA approves one exact three-stack set and every customer artifact is -published, run: +After a train's first release for one stack, and after its customer artifacts +are published, update the `compatibility:` block if the qualification result +changed, sync, then freeze that stack's docs alone: ```bash -go run -C tools/docs-version-sync . \ - --target main \ - --update-catalog \ - --qualification-version X.Y.Z \ - --stack-version A.B.C \ - --compute-stack-version D.E.F \ - --observability-stack-version G.H.I +go run -C tools/docs-version-sync . --target main --update-catalog go run -C tools/docs-version-sync . --target main -./tools/scripts/cut-docs-version.sh vX.Y.Z +./tools/scripts/cut-docs-version.sh --stack --train X.Y ``` -Qualification requires all three exact stack versions. The snapshot command -copies the generated docs and catalog. It updates the version dropdown with the -documentation version and all three stack versions. A later development sync -can move `docs/user/` forward without changing the versioned snapshot. +The freeze copies only that stack's documentation tree and catalog snapshot +and adds the train to that stack's version list. The other two stacks are not +touched. A later development sync moves `docs//` forward without +changing the frozen tree. See `tools/docs-version-sync/README.md` for the +exact flags. ## Compare Release Sets diff --git a/deploy/stacks/nvcf-compute-plane/README.md b/deploy/stacks/nvcf-compute-plane/README.md index 3097add920..8bd96ab8f2 100644 --- a/deploy/stacks/nvcf-compute-plane/README.md +++ b/deploy/stacks/nvcf-compute-plane/README.md @@ -171,11 +171,11 @@ Enabling `addons.kaiScheduler.enabled` or `addons.dynamoOperator.enabled` also adds the matching NVCA feature gate. Enabling KAI, Grove, or Dynamo permits their workload resource types in the NVCA validation policy. -See [Gang Scheduling](../../../docs/user/cluster-management/gang-scheduling.md) +See [Gang Scheduling](../../../docs/compute-plane/cluster-management/gang-scheduling.md) for atomic workload placement and -[Topology-Aware Scheduling](../../../docs/user/cluster-management/topology-aware-scheduling.md) +[Topology-Aware Scheduling](../../../docs/compute-plane/cluster-management/topology-aware-scheduling.md) for GPU clique placement. See -[KAI Scheduler](../../../docs/user/cluster-management/kai-scheduler.md) for +[KAI Scheduler](../../../docs/compute-plane/cluster-management/kai-scheduler.md) for queue configuration and standalone installation. ## Multi-Cluster Example diff --git a/docs/AGENTS.md b/docs/AGENTS.md index e81ae768a3..66180fdbee 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -10,38 +10,52 @@ external workspace index. ## Layout -- `docs/user/`: top-of-tree customer-facing documentation published as `dev`. -- `docs/v*/` and `docs/cp-*/`: frozen versioned documentation. Do not edit these trees unless the user explicitly asks for a historical docs fix. -- `docs/ngc-managed/`: legacy NGC-managed (BYOC) platform documentation. Separate from the self-hosted docs in `docs/user/`. +The published site is one Fern site with four products. Each stack product +has its own version menu. Overview is unversioned. + +- `docs/overview/`: unversioned shared documentation: compatibility matrix, quickstart, manifest, image mirroring, multi-tenancy, function usage (API, CLI, function and task creation, invocation, LLM gateway), load testing, release-notes index, local development, and shared assets under `images/` and `samples/`. +- `docs/self-managed/`: top-of-tree Self-Managed Stack (control plane) documentation published as `dev`. +- `docs/compute-plane/`: top-of-tree Compute Plane Stack documentation published as `dev`. +- `docs/observability/`: top-of-tree Observability Stack documentation published as `dev`. +- `docs/-/`: frozen per-stack documentation for a release train, for example `docs/observability-1.3/`. Do not edit these trees unless the user explicitly asks for a historical docs fix. `docs/self-managed-1.0/` is the full pre-split tree frozen at the 1.0.0 retag and contains compute-plane and observability pages as well. +- `docs/v*/`: frozen legacy full-tree documentation from before the per-stack split. Same rule: do not edit. +- `docs/ngc-managed/`: legacy NGC-managed (BYOC) platform documentation, published under Overview. - `docs/dev/`: developer and local workflow documentation. - `docs/version-catalog/main.yaml`: source of truth for generated artifact versions in top-of-tree docs. -- `fern/versions/dev.yml`: source of truth for top-of-tree docs navigation. -- `fern/versions/.yml`: source of truth for versioned docs navigation. +- `fern/docs.yml`: product and version registry. +- `fern/products/overview.yml`: Overview navigation. +- `fern/products//dev.yml`: top-of-tree navigation for one stack. +- `fern/products//.yml`: frozen navigation for one stack version. Legacy full-tree versions live under `fern/products/self-managed/`. + +A page belongs to exactly one product. Links inside a product stay relative. +Links to a page in another product use an absolute site path such as +`/nvcf/self-managed/installation-overview` or `/nvcf/overview/quickstart`, +because Fern resolves relative links inside the rendering product. ## Navigation Prefer the Fern navigation files and the filesystem over static route tables. -1. For top-of-tree docs, start with `fern/versions/dev.yml`. -2. For pinned release docs, use the matching file under `fern/versions/`. +1. For top-of-tree docs, start with `fern/products/overview.yml` or `fern/products//dev.yml`. +2. For pinned release docs, use the matching version file under `fern/products//`. 3. Confirm the mapped `path:` exists before answering. 4. If a nav item uses `href:`, treat it as an external page. Do not invent a local file. 5. If Fern nav does not answer the question, search with `rg`: ```bash -rg -n "" docs/user docs/dev docs/v* docs/cp-* +rg -n "" docs/overview docs/self-managed docs/compute-plane docs/observability docs/dev ``` Useful file listing commands: ```bash -rg --files docs/user docs/dev -rg --files docs/v* docs/cp-* +rg --files docs/overview docs/self-managed docs/compute-plane docs/observability docs/dev +rg --files docs/*-[0-9]* docs/v* ``` ## Editing -Use `docs/user/` for top-of-tree customer docs and `docs/dev/` for developer workflows. The default published docs route points to the latest stable version, not `docs/user/`. +Use the product tree that owns the page for top-of-tree customer docs and `docs/dev/` for developer workflows. Each stack product's default route points to that stack's latest frozen version once one exists, otherwise to `dev`. ### Artifact manifest @@ -49,7 +63,7 @@ For the maintainer workflow, including automatic stack releases, public publication updates, and new artifact registration, see [`tools/docs-version-sync/README.md`](../tools/docs-version-sync/README.md). -The generated tables in `docs/user/manifest.md` use catalog artifacts and +The generated tables in `docs/overview/manifest.md` use catalog artifacts and `manifest.entries` from `docs/version-catalog/main.yaml`. For each entry, set its deployment plane, kind, requirement, public-safe description, and public GitHub or upstream source links. Use `artifact_id` for catalog artifacts and @@ -102,11 +116,12 @@ exact public locations in `publications` and mark unavailable versions in `version` so charts, images, and resources with the same name remain distinct. Version overrides also require `name` and `type`. -After QA qualifies an exact three-stack combination, use -`--qualification-version` with all three explicit stack version flags. Generate -the docs, then run `tools/scripts/cut-docs-version.sh`. The versioned catalog -snapshot and Fern dropdown record the docs version and all three stack -versions. +Stacks qualify and freeze documentation independently. When a stack's release +train is qualified, run `tools/scripts/cut-docs-version.sh` for that stack and +train. It copies only that stack's tree, adds the version to that product's +menu in `fern/docs.yml`, and snapshots the catalog. The compatibility matrix in +`docs/overview/compatibility-matrix.md` is generated and stays current across +all stacks; it is not frozen. Generated blocks are marked with comments such as: diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/caches.md b/docs/compute-plane-1.0/caches.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/caches.md rename to docs/compute-plane-1.0/caches.md diff --git a/docs/user/cluster-management/configuration.md b/docs/compute-plane-1.0/cluster-management/configuration.md similarity index 99% rename from docs/user/cluster-management/configuration.md rename to docs/compute-plane-1.0/cluster-management/configuration.md index be39739853..ce24ae7d19 100644 --- a/docs/user/cluster-management/configuration.md +++ b/docs/compute-plane-1.0/cluster-management/configuration.md @@ -19,7 +19,7 @@ See below for descriptions of all available configuration options. | Priority Class | Set appropriate kubernetes priority class name for cluster agent and the operator pod. Additional details: [Priority Class](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) | | Model Cache Volume Mount Options | Configure the model cache volume mount options based on the CSI Driver capabilities on the cluster. Refer to the CSI Driver documentation. Defaults to `Enabled` and `ro,norecovery,nouuid` on an upgrade. Requires cluster reconfiguration after upgrade to prevent disruption.Additional details: [Mount options](https://man7.org/linux/man-pages/man8/mount.8.html) | | Network CIDR Range | Quoted & comma separated list of CIDR range for outbound network access for the infrastructure components & workloads on the cluster. | -| Worker Degradation Period | Stabilization time (in minutes) before cluster agent fails to consider a worker as healthy and initiates a purge. This also affects terminal worker failure timing for Helm functions that enable `StatusByWorkerReadiness`. See [Helm Functions](../helm-functions.md#use-worker-readiness-for-function-health). | +| Worker Degradation Period | Stabilization time (in minutes) before cluster agent fails to consider a worker as healthy and initiates a purge. This also affects terminal worker failure timing for Helm functions that enable `StatusByWorkerReadiness`. See [Helm Functions](/nvcf/self-managed/helm-functions#use-worker-readiness-for-function-health). | ## Cluster Features @@ -301,7 +301,7 @@ The NVCA operator requires outbound network connectivity to pull images, charts, 2. **Container Registry and NVCF Control Plane Access** - NVCA requires access to your container registry to pull images and Helm charts. - - NVCA requires network access to NVCF control plane services (SIS, NATS, ESS) running in your cluster. The specific endpoints depend on your gateway configuration. See [gateway-routing](../gateway-routing.md) for details. + - NVCA requires network access to NVCF control plane services (SIS, NATS, ESS) running in your cluster. The specific endpoints depend on your gateway configuration. See [gateway-routing](/nvcf/self-managed/gateway-routing) for details. 3. **Monitoring and Logging** @@ -859,7 +859,7 @@ agentConfig: `--quic-insecure` to the `pylon` sidecar. Use it only for local or isolated test clusters that run the LLM request router tunnel without TLS. For the full LLM addon setup, see -[LLM Function Enablement](../llm-function-enablement.md). +[LLM Function Enablement](/nvcf/self-managed/llm-function-enablement). BYOO collector debug and log chunking example: diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/container-cache.md b/docs/compute-plane-1.0/cluster-management/container-cache.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/container-cache.md rename to docs/compute-plane-1.0/cluster-management/container-cache.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/gang-scheduling.md b/docs/compute-plane-1.0/cluster-management/gang-scheduling.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/gang-scheduling.md rename to docs/compute-plane-1.0/cluster-management/gang-scheduling.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/gxcache.md b/docs/compute-plane-1.0/cluster-management/gxcache.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/gxcache.md rename to docs/compute-plane-1.0/cluster-management/gxcache.md diff --git a/docs/user/cluster-management/index.md b/docs/compute-plane-1.0/cluster-management/index.md similarity index 90% rename from docs/user/cluster-management/index.md rename to docs/compute-plane-1.0/cluster-management/index.md index 6fec3b862e..54d9037214 100644 --- a/docs/user/cluster-management/index.md +++ b/docs/compute-plane-1.0/cluster-management/index.md @@ -2,7 +2,7 @@ The NVIDIA Cluster Agent (NVCA) connects GPU clusters to the NVCF control plane, enabling them to act as deployment targets for Cloud Functions. NVCA is a function deployment orchestrator that registers a cluster's GPU resources, communicates with the control plane, and manages the lifecycle of function deployments on GPU nodes. -For a fresh install, use the [Quickstart](../quickstart.md). The one-click CLI flow can register a GPU cluster as part of the install. Use this section for manual cluster registration, standalone NVCA installation, and day-two cluster configuration. +For a fresh install, use the [Quickstart](/nvcf/overview/quickstart). The one-click CLI flow can register a GPU cluster as part of the install. Use this section for manual cluster registration, standalone NVCA installation, and day-two cluster configuration. If you pin NVCA separately from the recommended compute-plane stack, check the @@ -20,7 +20,7 @@ After installing NVCA on a cluster: | Key Type | Description | | --- | --- | -| NVCF API Key (NAK) | Used by NVCA to authenticate with the control plane. See [self-hosted-api](../api.md) for details on API key generation. | +| NVCF API Key (NAK) | Used by NVCA to authenticate with the control plane. See [self-hosted-api](/nvcf/self-managed/api) for details on API key generation. | ## Prerequisites diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/kai-scheduler.md b/docs/compute-plane-1.0/cluster-management/kai-scheduler.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/kai-scheduler.md rename to docs/compute-plane-1.0/cluster-management/kai-scheduler.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/model-cache.md b/docs/compute-plane-1.0/cluster-management/model-cache.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/model-cache.md rename to docs/compute-plane-1.0/cluster-management/model-cache.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/monitoring.md b/docs/compute-plane-1.0/cluster-management/monitoring.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/monitoring.md rename to docs/compute-plane-1.0/cluster-management/monitoring.md diff --git a/docs/user/cluster-management/nsight-profiling.md b/docs/compute-plane-1.0/cluster-management/nsight-profiling.md similarity index 97% rename from docs/user/cluster-management/nsight-profiling.md rename to docs/compute-plane-1.0/cluster-management/nsight-profiling.md index 7c6b1b1bef..f798a97a7d 100644 --- a/docs/user/cluster-management/nsight-profiling.md +++ b/docs/compute-plane-1.0/cluster-management/nsight-profiling.md @@ -233,7 +233,7 @@ cluster policy. Recreate existing function pods after the label is present. ## Run a Capture Use this flow with an existing Gemma-based LLM function, or create one using the -[LLM Gateway](../llm-gateway.md#function-configuration) function configuration +[LLM Gateway](/nvcf/self-managed/llm-gateway#function-configuration) function configuration pattern. Keep the model name aligned with the function's configured `models[].name` value. @@ -390,8 +390,8 @@ or profile in a maintenance window. ## See Also - [Self-Managed Clusters](./self-managed.md) -- [LLM Gateway](../llm-gateway.md) -- [Generic HTTP Function Invocation](../generic-http-function-invocation.md) +- [LLM Gateway](/nvcf/self-managed/llm-gateway) +- [Generic HTTP Function Invocation](/nvcf/self-managed/generic-http-function-invocation) - [NVIDIA Nsight Operator Installation Guide](https://docs.nvidia.com/nsight-operator/InstallationGuide/index.html) - [NVIDIA Nsight Operator User Guide](https://docs.nvidia.com/nsight-operator/UserGuide/index.html) - [NVIDIA Nsight Operator CRD Reference](https://docs.nvidia.com/nsight-operator/CRDReference/index.html) diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/reference.md b/docs/compute-plane-1.0/cluster-management/reference.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/reference.md rename to docs/compute-plane-1.0/cluster-management/reference.md diff --git a/docs/user/cluster-management/self-managed.md b/docs/compute-plane-1.0/cluster-management/self-managed.md similarity index 96% rename from docs/user/cluster-management/self-managed.md rename to docs/compute-plane-1.0/cluster-management/self-managed.md index 0eb2c6705b..728098a6ed 100644 --- a/docs/user/cluster-management/self-managed.md +++ b/docs/compute-plane-1.0/cluster-management/self-managed.md @@ -9,7 +9,7 @@ local ConfigMap and authenticates through the local OpenBao (Vault) instance. A running NVCF control plane (SIS, OpenBao, NATS, Cassandra, and all core -services) is required. The [Quickstart](../quickstart.md) can install the +services) is required. The [Quickstart](/nvcf/overview/quickstart) can install the control plane and register a GPU cluster in one flow. Use this page when you need to install or operate the NVCA Operator after using the Helmfile installation path. @@ -27,7 +27,7 @@ cd nvcf Before installing the NVCA Operator, ensure the following prerequisites are met: -- The [control plane](../helmfile-installation.md) is installed and all core services are running. +- The [control plane](/nvcf/self-managed/helmfile-installation) is installed and all core services are running. - The [NVIDIA GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) is installed on the GPU cluster. The GPU Operator manages the NVIDIA drivers, device plugin, and GPU feature discovery required for workload scheduling. For development or testing environments without physical GPUs, see [fake-gpu-operator](../fake-gpu-operator). @@ -37,10 +37,10 @@ Before installing the NVCA Operator, ensure the following prerequisites are met: [topology-aware scheduling](./topology-aware-scheduling.md) with Grove and Dynamo. -- `GPU Workload Components` must be available in a user-managed registry that your Kubernetes cluster can access. See `GPU Workload Components` under [self-hosted-artifact-manifest](../manifest.md) for necessary artifacts and [self-hosted-image-mirroring](../image-mirroring.md) for mirroring instructions. +- `GPU Workload Components` must be available in a user-managed registry that your Kubernetes cluster can access. See `GPU Workload Components` under [self-hosted-artifact-manifest](/nvcf/overview/manifest) for necessary artifacts and [self-hosted-image-mirroring](/nvcf/overview/image-mirroring) for mirroring instructions. - `nvcf-cli` is available on the deployment machine. The compute-plane Makefile - calls it during cluster registration. See [self-hosted-cli](../cli.md) for + calls it during cluster registration. See [self-hosted-cli](/nvcf/self-managed/cli) for CLI installation and configuration. - The [SMB CSI driver](https://github.com/kubernetes-csi/csi-driver-smb) (`smb.csi.k8s.io`) must be installed on the GPU cluster. It is required for NVCA shared model cache storage (samba sidecar). Install it with: @@ -145,15 +145,15 @@ the default values for the compute-plane install. Non-empty `revalServiceHostHeaderOverride`, `natsURL`, and `natsHostOverride` fields in the selected compute-plane environment take precedence. Verify the effective compute-reachable endpoints from those inputs resolve from the GPU cluster. See -[gateway-routing](../gateway-routing.md) for service DNS and TLS guidance. +[gateway-routing](/nvcf/self-managed/gateway-routing) for service DNS and TLS guidance. ## Register the cluster Register the GPU cluster with the control plane before installing the operator. The `nvcf-cli` discovers the cluster's OIDC issuer and JWKS and records them with the control plane, then returns the Helm values the operator needs. See -[self-hosted-cli](../cli.md) for CLI installation and configuration, and the -[Cluster Registration](../cli.md#cluster-registration) reference for full flag +[self-hosted-cli](/nvcf/self-managed/cli) for CLI installation and configuration, and the +[Cluster Registration](/nvcf/self-managed/cli#cluster-registration) reference for full flag and output details. @@ -586,7 +586,7 @@ The `instanceType` and `gpu` values depend on the GPU types available in your cl For invocation, the Host header uses wildcard subdomain routing: `.invocation.`. The URL path should match the function's `inferenceUrl` (e.g., `/echo`). For full HTTP invocation behavior, streaming, and errors, see -[Generic HTTP Function Invocation](../generic-http-function-invocation.md). +[Generic HTTP Function Invocation](/nvcf/self-managed/generic-http-function-invocation). @@ -595,7 +595,7 @@ You can also use the NVCF CLI for easier function management: - Create, deploy, and invoke functions with simple commands - Create or update registry credentials without manual API calls -See [self-hosted-cli](../cli.md) for installation and usage instructions. +See [self-hosted-cli](/nvcf/self-managed/cli) for installation and usage instructions. ## Re-registering a cluster @@ -696,7 +696,7 @@ deletion. If you encounter stuck resources, see [Handling Stuck Resources] below ### Handling Stuck Resources If step 1 times out and namespaces remain stuck in `Terminating` state, or function pods in -`nvcf-backend` prevent cleanup, use the [force-cleanup-script](../troubleshooting.md). This script removes +`nvcf-backend` prevent cleanup, use the [force-cleanup-script](/nvcf/self-managed/troubleshooting). This script removes finalizers on stuck NVCA resources, force-deletes function pods, and cleans up all NVCA namespaces. diff --git a/docs/user/cluster-management/topology-aware-scheduling.md b/docs/compute-plane-1.0/cluster-management/topology-aware-scheduling.md similarity index 98% rename from docs/user/cluster-management/topology-aware-scheduling.md rename to docs/compute-plane-1.0/cluster-management/topology-aware-scheduling.md index 12fc5adbfc..c2ff8c557c 100644 --- a/docs/user/cluster-management/topology-aware-scheduling.md +++ b/docs/compute-plane-1.0/cluster-management/topology-aware-scheduling.md @@ -23,7 +23,7 @@ and [Dynamo topology guide](https://docs.nvidia.com/dynamo/v1.4.1/kubernetes-dep for component-level details. Helm functions can also use the legacy -[`dra.nvcf.nvidia.io` partition annotation](../helm-functions.md#legacy-nvca-nvlink-partition-annotation). +[`dra.nvcf.nvidia.io` partition annotation](/nvcf/self-managed/helm-functions#legacy-nvca-nvlink-partition-annotation). That path uses Kubernetes Pod affinity and is best-effort without KAI Scheduler or Grove topology-aware scheduling. Use the KAI or Grove mechanisms on this page when clique placement must be coordinated for the complete workload. diff --git a/docs/compute-plane-1.0/fake-gpu-operator.md b/docs/compute-plane-1.0/fake-gpu-operator.md new file mode 100644 index 0000000000..4621f974d3 --- /dev/null +++ b/docs/compute-plane-1.0/fake-gpu-operator.md @@ -0,0 +1,222 @@ +# Fake GPU Operator (Development / Testing) + +For development, staging, load testing, or CI environments that lack physical NVIDIA GPUs, +you can install a fake GPU operator to simulate GPU resources on cluster nodes. This allows +the NVCA agent to discover GPUs and manage function deployments without actual GPU hardware. + + +The fake GPU operator is for **non-production use only**. For production deployments with +real GPUs, install the +[NVIDIA GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/). + + + +## Prerequisites + +- A running Kubernetes cluster with `kubectl` access +- `helm` >= 3.12 + +### Install KWOK + +The fake GPU operator depends on [KWOK (Kubernetes Without Kubelet)](https://kwok.sigs.k8s.io/) +to manage simulated GPU device plugins on nodes. Install KWOK before the fake GPU operator: + +```bash +kubectl apply -f https://github.com/kubernetes-sigs/kwok/releases/download/v0.7.0/kwok.yaml +``` + +Verify the KWOK controller is running: + +```bash +kubectl get pods -n kube-system -l app=kwok-controller +# Expected: kwok-controller-... 1/1 Running +``` + + +The KWOK install may produce a FlowSchema error +(`creation or update of FlowSchema object ... is not allowed`). +This is non-critical and can be safely ignored. + + + +## Installation + +Add the RunAI helm repository and install the fake GPU operator: + +```bash +helm repo add fake-gpu-operator \ + https://runai.jfrog.io/artifactory/api/helm/fake-gpu-operator-charts-prod \ + --force-update + +helm upgrade -i gpu-operator fake-gpu-operator/fake-gpu-operator \ + -n gpu-operator --create-namespace \ + --set 'topology.nodePools.default.gpuCount=8' \ + --set 'topology.nodePools.default.gpuProduct=NVIDIA-H100-80GB-HBM3' +``` + +This configures one node pool named `default` with 8 simulated H100 GPUs per node. + + +**topology.nodePools must be a map, not an array.** + +Using array index syntax (`--set 'topology.nodePools[0].gpuCount=8'`) will create a +YAML array instead of a map and cause the status-updater to fail with: + +```text +yaml: unmarshal errors: cannot unmarshal !!seq into map[string]topology.NodePoolTopology +``` + +Always use named keys: `topology.nodePools.default.gpuCount=8`. + + + +## Node Labeling + +The fake GPU operator watches for nodes with the label +`run.ai/simulated-gpu-node-pool=` and patches their status to advertise +fake `nvidia.com/gpu` extended resources. You must label the nodes that should receive +simulated GPUs: + +```bash +kubectl label node run.ai/simulated-gpu-node-pool=default +``` + +The pool name (`default`) must match a key in `topology.nodePools` from the helm install. + +### GPU Metadata Labels (Optional) + +The NVCA agent uses several GPU metadata labels for dynamic discovery. On real GPU nodes +these are set by the NVIDIA GPU Operator. To suppress warnings from NVCA on fake GPU nodes, +add the following labels: + +```bash +kubectl label node \ + nvidia.com/gpu.family=hopper \ + nvidia.com/gpu.machine=NVIDIA-DGX-H100 \ + nvidia.com/cuda.driver.major=535 \ + --overwrite +``` + +Adjust the values to match the GPU product you configured (e.g., `ampere` for A100, +`ada` for L40S). + +## RuntimeClass Ownership Conflict + +The fake GPU operator chart creates `RuntimeClass/nvidia` and several namespaced resources in `gpu-operator`. Helm fails with `invalid ownership metadata` if one of those objects already exists and is not owned by release `gpu-operator` in namespace `gpu-operator`. + +For local k3d development, the recovery workflow is to rerun `make build-and-deploy-cluster` in `tools/ncp-local-cluster/`, which removes known stale fake GPU operator resources without deleting the cluster. For manual chart debugging, inspect ownership before deleting anything. If another Helm release owns the resource, remove that release instead of deleting the resource directly. + +## Verification + +Check that the fake GPU operator pods are running: + +```bash +kubectl get pods -n gpu-operator +# Expected: 3 pods Running (topology-server, status-updater, kwok-gpu-device-plugin) +``` + +Confirm that labeled nodes now advertise GPU resources: + +```bash +kubectl get nodes -o custom-columns="NAME:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu" +# Labeled nodes should show the configured GPU count (e.g., 8) +``` + +If GPUs do not appear, verify the node has the `run.ai/simulated-gpu-node-pool=default` +label and that the status-updater pod is not in an error state. + +## Integration with NVCF + +### Recommended Installation Order + +For the smoothest experience, install the fake GPU operator **before** running +`helmfile sync`. This way the NVCA agent discovers GPUs on its first boot and no +re-registration is needed. + +The recommended sequence is: + +1. Install KWOK +2. Install fake-gpu-operator and label target nodes +3. Verify `nvidia.com/gpu` appears in node allocatable resources +4. Proceed with the [control-plane installation](/nvcf/self-managed/helmfile-installation) + +### If Installed After the Control Plane + +If you add the fake GPU operator to a cluster that already has NVCF deployed, the NVCA +agent will be crash-looping because it cannot find GPUs. After installing the fake GPU +operator and verifying GPUs appear on nodes, re-register the cluster and restart the +operator: + +```bash +# Re-run the cluster bootstrap +kubectl exec -n nvca-operator deploy/nvca-operator -c nvca-operator -- \ + /usr/bin/nvca-self-managed bootstrap --system-namespace nvca-operator + +# Restart the operator (it caches cluster IDs at startup) +kubectl rollout restart deployment nvca-operator -n nvca-operator +kubectl rollout status deployment nvca-operator -n nvca-operator --timeout=120s +``` + +The operator restart will re-run the bootstrap init container, recreate the NVCFBackend +resource, and spawn a fresh NVCA agent pod that discovers the simulated GPUs. + +For details on the bootstrap process, see [Self-Managed Clusters](/nvcf/compute-plane/self-managed-clusters) (Manual Cluster +Registration). + +## Customization + +### GPU Count and Product + +Adjust the GPU count, product name, and memory per node pool: + +```bash +helm upgrade gpu-operator fake-gpu-operator/fake-gpu-operator \ + -n gpu-operator \ + --set 'topology.nodePools.default.gpuCount=4' \ + --set 'topology.nodePools.default.gpuProduct=NVIDIA-A100-SXM4-80GB' \ + --set 'topology.nodePools.default.gpuMemory=81920' +``` + +### Multiple Node Pools + +Define multiple pools with different GPU configurations by using different map keys: + +```bash +helm upgrade gpu-operator fake-gpu-operator/fake-gpu-operator \ + -n gpu-operator \ + --set 'topology.nodePools.h100-pool.gpuCount=8' \ + --set 'topology.nodePools.h100-pool.gpuProduct=NVIDIA-H100-80GB-HBM3' \ + --set 'topology.nodePools.a100-pool.gpuCount=4' \ + --set 'topology.nodePools.a100-pool.gpuProduct=NVIDIA-A100-SXM4-80GB' +``` + +Then label nodes with the corresponding pool name: + +```bash +kubectl label node run.ai/simulated-gpu-node-pool=h100-pool +kubectl label node run.ai/simulated-gpu-node-pool=a100-pool +``` + +## Teardown + +To remove the fake GPU operator and all simulated GPU resources: + +```bash +# Remove the fake GPU operator +helm uninstall gpu-operator -n gpu-operator +kubectl delete namespace gpu-operator --ignore-not-found + +# Remove KWOK +kubectl delete -f https://github.com/kubernetes-sigs/kwok/releases/download/v0.7.0/kwok.yaml + +# Remove the node labels (for each labeled node) +kubectl label node run.ai/simulated-gpu-node-pool- +kubectl label node nvidia.com/gpu.product- +kubectl label node nvidia.com/gpu.family- +kubectl label node nvidia.com/gpu.machine- +kubectl label node nvidia.com/cuda.driver.major- +``` + +After removing the fake GPU operator, the NVCA agent will lose GPU visibility and begin +crash-looping. Either install a real GPU Operator with physical GPUs or uninstall the +NVCA operator. diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/caches.md b/docs/compute-plane-1.0/runbooks/caches.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/caches.md rename to docs/compute-plane-1.0/runbooks/caches.md diff --git a/docs/user/caches.md b/docs/compute-plane/caches.md similarity index 100% rename from docs/user/caches.md rename to docs/compute-plane/caches.md diff --git a/docs/compute-plane/cluster-management/configuration.md b/docs/compute-plane/cluster-management/configuration.md new file mode 100644 index 0000000000..ce24ae7d19 --- /dev/null +++ b/docs/compute-plane/cluster-management/configuration.md @@ -0,0 +1,1155 @@ +# NVCA Configuration + +This page documents NVCA configuration options. For cluster registration and lifecycle +operations, see [Self-Managed Clusters](./self-managed.md). + +## Advanced Settings + + +Some of the options below are supported only on Cluster Agent Versions `2.50.0` or higher. + + + +See below for descriptions of all available configuration options. + +| Configuration | Description | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Cluster Agent Version | Version of the cluster agent to be installed on the cluster. Defaults to the latest available. Recommended to use the latest version available at the time of registration unless there are business reasons to pick another version. | +| Node Selector Key and Node Selector Value | This Key-Value pair is the label selector key to control the placement of the cluster agent and the cluster agent operator pods to specific nodes on the cluster. Not providing a value will allow these infrastructure components to be placed anywhere on the cluster. Ensure there are matching nodes in the cluster using `kubectl get node -l key=value` before registration as incorrect value will cause operational issues. For additional details: [Labels & Selectors](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors) | +| Priority Class | Set appropriate kubernetes priority class name for cluster agent and the operator pod. Additional details: [Priority Class](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) | +| Model Cache Volume Mount Options | Configure the model cache volume mount options based on the CSI Driver capabilities on the cluster. Refer to the CSI Driver documentation. Defaults to `Enabled` and `ro,norecovery,nouuid` on an upgrade. Requires cluster reconfiguration after upgrade to prevent disruption.Additional details: [Mount options](https://man7.org/linux/man-pages/man8/mount.8.html) | +| Network CIDR Range | Quoted & comma separated list of CIDR range for outbound network access for the infrastructure components & workloads on the cluster. | +| Worker Degradation Period | Stabilization time (in minutes) before cluster agent fails to consider a worker as healthy and initiates a purge. This also affects terminal worker failure timing for Helm functions that enable `StatusByWorkerReadiness`. See [Helm Functions](/nvcf/self-managed/helm-functions#use-worker-readiness-for-function-health). | + +## Cluster Features + +Cluster Features allow enabling specific features on the cluster. Dynamic GPU Discovery is enabled by default. + +See below for descriptions of all cluster features. + +| Capability | Description | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Dynamic GPU Discovery | Enables automatic detection and management of allocatable GPU capacity within the cluster via the NVIDIA GPU Operator. This capability is **strongly recommended** and would only be disabled in cases where [Manual Instance Configuration](./configuration.md) is required. | +| Caching Support | Enhances application performance by storing frequently accessed data (models, resources and containers) in a cache. See [cluster-caching](./configuration.md). | +| Optimized AI Workload Scheduling | Enables KAI Scheduler for GPU bin-packing and queues. See [KAI Scheduler](./kai-scheduler.md), [Gang Scheduling](./gang-scheduling.md), and [Topology-Aware Scheduling](./topology-aware-scheduling.md). | +| Shared Cluster mode | Partitions the Kubernetes cluster's nodes into NVCF and non-NVCF pools. Set the label `nvca.nvcf.nvidia.io/schedule=true` on all nodes that can receive NVCF workload Pods. **Note**: this is an advanced use case and should not be used unless absolutely necessary | + + +Removing the Dynamic GPU Discovery will require manual instance configuration. See [Manual Instance Configuration](./configuration.md). + + + +### Caching Support + +Enabling caching for models, resources and containers is recommended for optimal performance. You must create `StorageClass` configurations for caching within your cluster to fully enable "Caching Support" with the Cluster Agent. See examples below. + + +Caching is not supported for Multi-Node Helm functions. If you attempt to deploy a Multi-Node Helm function with Caching Support enabled, the deployment will fail. + +Caching is also currently not supported for AWS EKS. + + + +**StorageClass Configurations in GCP** + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: nvcf-sc +provisioner: pd.csi.storage.gke.io +allowVolumeExpansion: true +volumeBindingMode: Immediate +reclaimPolicy: Retain +parameters: + type: pd-ssd + csi.storage.k8s.io/fstype: xfs +``` + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: nvcf-cc-sc +provisioner: pd.csi.storage.gke.io +allowVolumeExpansion: true +volumeBindingMode: Immediate +reclaimPolicy: Retain +parameters: + type: pd-ssd + csi.storage.k8s.io/fstype: xfs +``` + + +GCP currently allows only [10 VM's](https://cloud.google.com/compute/docs/disks#:~:text=You%20can%20attach%20a%20balanced,VMs%20in%20read%2Donly%20mode) to mount a Persistent Volume in Read-Only mode. + + + +**StorageClass Configurations in Azure** + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: nvcf-sc +provisioner: file.csi.azure.com +allowVolumeExpansion: true +volumeBindingMode: Immediate +reclaimPolicy: Retain +parameters: + skuName: Standard_LRS + csi.storage.k8s.io/fstype: xfs +``` + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: nvcf-cc-sc +provisioner: file.csi.azure.com +allowVolumeExpansion: true +volumeBindingMode: Immediate +reclaimPolicy: Retain +parameters: + skuName: Standard_LRS + csi.storage.k8s.io/fstype: xfs +``` + +**StorageClass Configurations in Oracle Cloud** + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: nvcf-sc +provisioner: blockvolume.csi.oraclecloud.com +allowVolumeExpansion: true +volumeBindingMode: Immediate +reclaimPolicy: Retain +parameters: + csi.storage.k8s.io/fstype: xfs +``` + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: nvcf-cc-sc +provisioner: blockvolume.csi.oraclecloud.com +allowVolumeExpansion: true +volumeBindingMode: Immediate +reclaimPolicy: Retain +parameters: + csi.storage.k8s.io/fstype: xfs +``` + +**Apply the StorageClass Configurations** + +Save the StorageClass template to files `nvcf-sc.yaml` and `nvcf-cc-sc.yaml` and apply them as: + +```bash +kubectl create -f nvcf-sc.yaml +kubectl create -f nvcf-cc-sc.yaml +``` + +**Override the Default Mount Options for Cache Volumes** + + +Supported in Cluster Agent Versions 2.45.21 or higher + + + + +Please note this is a Post NVCA Install Operation and needs careful consideration to ensure there are no volume corruptions. Use with caution. + + + +Cluster Agent with caching support by default will enable linux mount-options with `ro,norecovery,nouuid`. + +If the CSI Driver in the cluster doesn't support mount options then you may apply the following command on the cluster to disable the mount options + +```bash +nvcf_cluster_name="$(kubectl get nvcfbackends -n nvca-operator -o name | cut -d'/' -f2)" +kubectl patch nvcfbackends.nvcf.nvidia.io -n nvca-operator "$nvcf_cluster_name" \ + --type='merge' \ + -p '{"spec":{"overrides":{"agentConfig":{"cacheMountOptionsEnabled":false}}}}' +``` + +If you want to update the mount-options to a different value for example: `ro,norecovery`. You may use the following command. Replace these options with desired value as dictated by CSI Driver Volume Mount Options. + +```bash +nvcf_cluster_name="$(kubectl get nvcfbackends -n nvca-operator -o name | cut -d'/' -f2)" +kubectl patch nvcfbackends.nvcf.nvidia.io -n nvca-operator "$nvcf_cluster_name" \ + --type='merge' \ + -p '{"spec":{"overrides":{"agentConfig":{"cacheMountOptionsEnabled":true,"cacheMountOptions":"ro,norecovery"}}}}' +``` + +### Cluster Maintenance Modes + +The Cluster Agent supports two maintenance modes that control how workloads are handled during cluster configuration changes. Configure maintenance mode via the feature flags `CordonMaintenance` or `CordonAndDrainMaintenance` respectively. + +**Cordon Maintenance** + +In this mode, existing workloads continue to run uninterrupted on the cluster. New workloads will not be scheduled until maintenance mode is cleared. + +**Cordon and Drain Maintenance** + +In this mode, all existing workloads in the cluster are terminated. No updates to the state of workloads will be effective while in this mode. + + +Once maintenance mode is configured, it can take up to 10 minutes for the agent reconfiguration to take effect. + + + +### Host-Isolated Clusters + +Clusters with the `HostIsolation` attribute ensure that no two function or task instances run on the same node at the same time. Each node is dedicated to a single active workload instance. + + +`HostIsolation` and `AccountIsolation` are mutually exclusive. Enabling both attributes on the same cluster is not supported. Use `AccountIsolation` if you need isolation at the NCA account boundary; use `HostIsolation` if you need isolation at the individual function boundary. + + + +### Account-Isolated Clusters + + +Supported in Cluster Agent Versions 2.49.0 or higher + + + +Clusters with the `AccountIsolation` attribute have enhanced isolation between workloads, ensuring that function and task instances run on nodes isolated by NCAId. This is particularly important for customers with strict security requirements or those who want to ensure complete separation of workloads at the account level. + + +In Account Isolated mode, the cluster might be inefficient in GPU utilization if workloads are not designed to utilize the full capacity of the isolated nodes. +While toggling this attribute, the cluster workloads also have to be drained using [CordonAndDrainMaintenance](./configuration.md) mode to effectively re-balance the workloads as the attribute will not be applied retroactively. + + + +### NVLink-optimized Clusters + +Clusters with [MNNVL](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/dra-cds.html#dra-docs-compute-domains) GPUs like [GB200](https://www.nvidia.com/en-us/data-center/gb200-nvl72/) can run multi-node workloads that require inter-GPU data transfer with [large performance improvements](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/dra-cds.html#usage-example-a-multi-node-nvbandwidth-test) when properly configured. +The Cluster Agent can be directed to configure multi-node workloads with their own [ComputeDomains](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/dra-cds.html#computedomains-multi-node-nvlink-simplified) automatically to optimize inter-GPU connections. + +Additional prerequisites: +\- The [NVIDIA GPU DRA driver](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/dra-intro-install.html) must be installed. +\- The `NVLinkOptimized` cluster attribute must be added during cluster registration. + +See [Topology-Aware Scheduling](./topology-aware-scheduling.md) to place +multi-node workloads in one GPU clique. Use +[Gang Scheduling](./gang-scheduling.md) when every Pod must be placed +atomically. + + +In NVLink-optimized mode, the NVIDIA GPU DRA driver currently limits one GPU-enabled Pod to a node. To optimally utilize these clusters, GPU-enabled Pods _should_ request a full node's worth of GPUs. For example, nodes in GB200 clusters have 4 GPUs each so all containers and all GPU-enabled Pods in a workload must request `nvidia.com/gpu` values that sum to 4. + + + +### Kata Container-Isolated Workloads + +Clusters that have this attribute run all function/task Pods in [Kata Containers](https://katacontainers.io/) without exception. + +Additional cluster restrictions to be aware of: + +- Pod containers _must_ at least have resource `limits` defined for `cpu` and `memory`. If unset, runtime behavior is undefined. + +- Object count limits are configured for resource fairness in these clusters: + + - ConfigMaps: 20 + - Secrets: 20 + - Services: 20 + - Pods: 100 + - Jobs: 10 + - CronJobs: 10 + - Deployments: 10 + - ReplicaSets: 10 + - StatefulSets: 10 + +## Network Configuration + + +The network policies described in this section are only enforced if your cluster's Container Network Interface (CNI) supports Kubernetes Network Policies. Common CNIs that support network policies include: + +- Calico +- Cilium +- Weave Net +- Antrea + +If your cluster uses a CNI that doesn't support network policies, the security controls described below will not be enforced, and pods will be able to communicate with each other without restrictions. This could lead to security vulnerabilities. + + + +The NVCA operator requires outbound network connectivity to pull images, charts, and report logs and metrics. During installation, the operator pre-configures the `nvca-namespace-networkpolicies` configmap with the following network policies: + +| Policy Name | Description | +| ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| allow-egress-gxcache | Allows egress traffic to the GX Cache namespace for caching operations (only relevant for NVIDIA managed clusters) | +| allow-egress-internet-no-internal-no-api | Allows egress traffic to the public internet (0.0.0.0/0) but blocks traffic to common private IP ranges. Also allows DNS resolution via kube-dns. | +| allow-egress-intra-namespace | Allows pod-to-pod communication within the same namespace. Applied only to per-instance function namespaces (for example a MiniService's utils pod reaching its own inference pod), never to the shared `nvcf-backend` namespace. | +| allow-egress-nvcf-cache | Allows egress traffic to NVCF cache services (only relevant for NVIDIA managed clusters) | +| allow-egress-prometheus-nvcf-byoo | Allows egress traffic to Prometheus monitoring endpoints (only relevant for NVIDIA managed clusters) | +| allow-ingress-monitoring | Allows ingress from the `monitoring` namespace on supported monitoring ports. In per-instance function namespaces, also allows same-namespace ingress (paired with allow-egress-intra-namespace); same-namespace ingress is not added to the shared `nvcf-backend` namespace. | +| allow-ingress-monitoring-dcgm | Allows ingress traffic for DCGM monitoring | +| allow-ingress-monitoring-gxcache | Allows ingress traffic for GX Cache monitoring (only relevant for NVIDIA managed clusters) | + +## Key Network Requirements + +1. **Kubernetes API Access** + + - NVCA requires access to the Kubernetes API + - Consult your cloud provider's documentation (e.g., Azure, AWS, GCP) for the Kubernetes API endpoint + +2. **Container Registry and NVCF Control Plane Access** + + - NVCA requires access to your container registry to pull images and Helm charts. + - NVCA requires network access to NVCF control plane services (SIS, NATS, ESS) running in your cluster. The specific endpoints depend on your gateway configuration. See [gateway-routing](/nvcf/self-managed/gateway-routing) for details. + +3. **Monitoring and Logging** + + - If your environment requires advanced monitoring or logging (e.g., sending logs to external endpoints), ensure your cluster's NetworkPolicy or firewall rules allow egress to the required monitoring/logging domains + +## Network Policy Customization via ConfigMap + +The NVCA operator pre-configures the `nvca-namespace-networkpolicies` configmap during installation. If you need to customize these policies for your cluster, you can use a configmap to override the default policies. + +To customize a network policy: + +1. Create a configmap with your custom network policy, for example: + + ```yaml + + apiVersion: v1 + kind: ConfigMap + metadata: + name: demopatch-configmap + namespace: nvca-operator + labels: + nvca.nvcf.nvidia.io/operator-kustomization: enabled + data: + patches: | + - target: + group: "" + version: v1 + kind: ConfigMap + name: nvca-namespace-networkpolicies + patch: |- + - op: replace + path: /data/allow-egress-internet-no-internal-no-api + value: | + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: allow-egress-internet-no-internal-no-api + labels: + app.kubernetes.io/name: nvca + app.kubernetes.io/instance: nvca + app.kubernetes.io/version: "1.0" + app.kubernetes.io/managed-by: nvca-operator + spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: gxcache + ports: + - port: 8888 + protocol: TCP + - port: 8889 + protocol: TCP + + ``` + +2. Apply the configmap: + + ```bash + + kubectl apply -f patchcm.yaml + ``` + +3. Verify the changes: + + ```bash + + kubectl logs -n nvca-operator -l app.kubernetes.io/name=nvca-operator + + ``` + + You should see a message indicating successful patching: + `configmap patched successfully` + +The changes will be applied to the `nvcf-backend` namespace and will be used for all new namespaces' network policies. The network policies will also be updated across all helm chart namespaces. + +## Network Policy Customization via clusterNetworkCIDRs Flag + +You can customize the `allow-egress-internet-no-internal-no-api` policy with helm, by adding on the `networkPolicy.clusterNetworkCIDRs` flag. For example: + +```bash +helm upgrade nvca-operator -n nvca-operator --create-namespace -i --reuse-values --wait \ + oci://${REGISTRY}/${REPOSITORY}/nvca-operator --version \ + --set networkPolicy.clusterNetworkCIDRs="{10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,100.64.0.0/12}" +``` + +This command will override the default k8s networking CIDRs specified in the `allow-egress-internet-no-internal-no-api` with your input. + +## Advanced: Additional Configuration Options + +## CSI Volume Mount Options + +The NVIDIA Cluster Agent supports customizing CSI volume mount options for caching. This allows you to configure specific mount options for the CSI volumes used in your cluster. + + +CSI volume mount options configuration is an **experimental feature** and may be subject to change in future releases. + + + +To configure CSI volume mount options: + +1. Get the NVCF cluster name: + +```bash +nvcf_cluster_name="$(kubectl get nvcfbackends -n nvca-operator -o name | cut -d'/' -f2)" +``` + +1. View current mount options configuration: + +```bash +kubectl get nvcfbackend -n nvca-operator "$nvcf_cluster_name" -o yaml | grep -A 5 "MountOptions" +``` + +1. Set mount options (example): + +```bash +kubectl patch nvcfbackends.nvcf.nvidia.io -n nvca-operator "$nvcf_cluster_name" \ + --type='merge' \ + -p '{"spec":{"overrides":{"agentConfig":{"cacheMountOptionsEnabled":true,"cacheMountOptions":"ro,norecovery,nouuid"}}}}' +``` + +1. Verify the changes: + +```bash +kubectl get nvcfbackend -n nvca-operator "$nvcf_cluster_name" -o yaml | grep -A 5 "MountOptions" +``` + +The default mount options are: +\- `ro`: Read-only mount +\- `norecovery`: Skip journal recovery +\- `nouuid`: Ignore filesystem UUID + +You can modify these options based on your specific requirements. The configuration will be applied to all CSI volumes created by the NVIDIA Cluster Agent for caching purposes. + +## Node Selection for Cloud Functions + +By default, the cluster agent uses all nodes discovered with GPU resources to schedule Cloud Functions and there are no additional configuration required. + +In order to limit the nodes that can run Cloud Functions, you may use `nvca.nvcf.nvidia.io/schedule=true` label on the specific nodes. + +If there are no nodes in the cluster with the `nvca.nvcf.nvidia.io/schedule=true` label set, the cluster agent will switch to the default behavior of using all nodes with GPUs. + +For example, to mark specific nodes as schedulable in a cluster: + +```bash +kubectl label node nvca.nvcf.nvidia.io/schedule=true +``` + +To mark a single node from the above set as unschedulable for nvcf workloads, you can unlabel using: + +```bash +kubectl label node nvca.nvcf.nvidia.io/schedule- +``` + +## GPU Product Name Override + +The NVIDIA Cluster Agent supports GPU product name override via node label. This is useful for customers who want to use a custom product name or override the default GPU product name. + +For example, to set the GPU product name for a node, use the following command: + +```bash +kubectl label node nvca.nvcf.nvidia.io/gpu.product= +``` + + +The GPU Product Name Override via node labeling only takes effect when there are no pre-existing active instances in the cluster. If active instances already exist with the original GPU instance types, the override will not be applied. + + +## Single MIG Mode Support + +The NVIDIA Cluster Agent supports GPUs configured in single +[Multi-Instance GPU (MIG)](https://docs.nvidia.com/datacenter/tesla/mig-user-guide/) +mode. When the NVIDIA GPU Operator is configured with the `single` MIG strategy, +each GPU is partitioned into multiple MIG instances of the same profile. NVCA +automatically detects MIG profiles from the `nvidia.com/gpu.product` node label and +generates a unique instance type for each profile. + +### How It Works + +In single MIG mode, the GPU Operator appends the MIG profile to the GPU product name +in the node label. For example: + +```text +nvidia.com/gpu.product=NVIDIA-H200-MIG-7g.141gb +nvidia.com/gpu.product=NVIDIA-RTX-PRO-6000-Blackwell-Server-Edition-MIG-2g.48gb +nvidia.com/gpu.product=NVIDIA-RTX-PRO-6000-Blackwell-Server-Edition-MIG-1g.24gb +``` + +NVCA parses the MIG suffix from the product name and produces a unique instance type +for each profile. The MIG profile in the instance type name uses hyphens and lowercase +(for example, `MIG-2g-48gb`). + +| GPU Product Label | Instance Type Name | +| --- | --- | +| `NVIDIA-H200-MIG-7g.141gb` | `NCP.GPU.H200-MIG-7g-141gb` | +| `NVIDIA-RTX-PRO-6000-...-MIG-2g.48gb` | `ON-PREM.GPU.RTXPRO6000-MIG-2g-48gb` | +| `NVIDIA-RTX-PRO-6000-...-MIG-1g.24gb` | `ON-PREM.GPU.RTXPRO6000-MIG-1g-24gb` | + +The cluster provider prefix, for example `NCP`, `ON-PREM`, or `AWS`, is determined +by the cluster registration configuration. + +### Prerequisites + +- **GPU Operator** configured with MIG strategy `single`. See the + [GPU Operator MIG documentation](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/gpu-operator-mig.html) + for setup instructions. +- **Dynamic GPU Discovery** enabled, the default for customer-managed clusters. +- NVCA **v2.53.0** or later. + + +Changing MIG mode or MIG profiles on a node is a disruptive operation. Nodes may +reboot or terminate GPU clients while applying the new MIG geometry. Before +reconfiguring MIG: + +1. **Cordon** the target nodes to prevent new workload scheduling. +2. **Drain** all active GPU workloads from the nodes. +3. Apply the MIG configuration change. +4. Wait for the GPU Operator MIG Manager to finish, see verification below. +5. **Uncordon** the nodes once verification passes. + +See the [GPU Operator MIG reconfiguration guide](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/gpu-operator-mig.html#configuring-mig-profiles) +for details on the reconfiguration process. + + + +### Verification + +After configuring MIG on your cluster, first confirm the GPU Operator has finished +applying the MIG profile before checking NVCA labels. + +**Step 1: Verify MIG Manager has converged** + +Check that MIG-configured nodes report `nvidia.com/mig.config.state=success`: + +```bash +# Check MIG configuration state on MIG nodes +kubectl get nodes -l nvidia.com/mig.strategy=single \ + -o custom-columns='NODE:.metadata.name,MIG_CONFIG:.metadata.labels.nvidia\.com/mig\.config,STATE:.metadata.labels.nvidia\.com/mig\.config\.state' +``` + +All target nodes should show `success` in the STATE column. If any node shows +`pending`, `rebooting`, or a failure state, wait for the GPU Operator to finish +or investigate the MIG Manager logs before proceeding. + +**Step 2: Verify NVCA instance type labels** + +Once MIG Manager has converged, verify that NVCA has detected the MIG profiles and +applied the correct instance type labels: + +```bash +# Check the GPU product label on MIG nodes +kubectl get nodes -l nvidia.com/mig.strategy=single \ + -o custom-columns='NODE:.metadata.name,GPU_PRODUCT:.metadata.labels.nvidia\.com/gpu\.product' + +# Verify the instance type label assigned by NVCA +kubectl get nodes -l nvidia.com/mig.strategy=single \ + -o custom-columns='NODE:.metadata.name,INSTANCE_TYPE:.metadata.labels.nvca\.nvcf\.nvidia\.io/instance-type' +``` + +Each MIG profile should have a distinct instance type label. For example, a node with +`MIG-2g.48gb` should show `ON-PREM.GPU.RTXPRO6000-MIG-2g-48gb` while a node with +`MIG-1g.24gb` should show `ON-PREM.GPU.RTXPRO6000-MIG-1g-24gb`. + + +Only the `single` MIG strategy is supported. In single mode, all GPUs on a node +are partitioned with the same MIG profile. The `mixed` MIG strategy, with different +profiles on the same node, is not currently supported by NVCA. + + + +## Managing Feature Flags + +The NVIDIA Cluster Agent supports various feature flags that can be enabled or disabled to customize its behavior. The following are some commonly used feature flags: + +| Feature Flag | Description | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| DynamicGPUDiscovery | Dynamically discover GPUs and instance types on this cluster. This is enabled by default for customer-managed clusters. | +| HelmSharedStorage | Configure Helm functions and tasks with shared read-only storage for ESS secrets. This is required for enabling Helm-based tasks in your cluster. Please note turning on this feature flag requires additional configuration, see [Helm Shared Storage section below](./configuration.md). | +| LogPosting | Post instance logs to SIS directly. This is enabled by default for NVIDIA managed clusters. | +| MultiNodeWorkloads | Instruct NVCA to report multi-node instance types to SIS during registration. | +| SelfHosted | Enables local vault-based authentication for self-hosted deployments. Required when `ngcConfig.clusterSource` is `self-managed`. | +| HelmAllowCPUNodes | Allow CPU-only pods (e.g. etcd, redis, envoy) from Helm-based functions to be scheduled on non-GPU nodes; GPU pods keep required instance-type affinity. Reduces cost and improves GPU utilization. Mutually exclusive with HelmResourceConstraints. See [helm-allow-cpu-nodes](./configuration.md). | + +### Setting Feature Flags at Install Time + +Feature flags can be set during the initial NVCA Operator installation through Helm values. + +**Standalone Helm** + +Set `selfManaged.featureGateValues` in your values file. The chart default is +`["DynamicGPUDiscovery"]`. + +**In the values file:** + +```yaml +selfManaged: + featureGateValues: ["DynamicGPUDiscovery", "SelfHosted", "LogPosting"] +``` + +**Or via** `--set` **during install:** + +```bash +helm upgrade --install nvca-operator \ + oci://${REGISTRY}/${REPOSITORY}/nvca-operator \ + --version 1.2.7 \ + --namespace nvca-operator --create-namespace \ + -f nvca-operator-values.yaml \ + --set 'selfManaged.featureGateValues={DynamicGPUDiscovery,SelfHosted,LogPosting}' +``` + + +The `--set` flag **replaces** the entire list. You must include all desired flags, +not just the new one. + + + +To update flags on an existing installation, run `helm upgrade` with the updated values +file or `--set`: + +```bash +helm upgrade nvca-operator \ + oci://${REGISTRY}/${REPOSITORY}/nvca-operator \ + --version 1.2.7 \ + --namespace nvca-operator \ + -f nvca-operator-values.yaml \ + --set 'selfManaged.featureGateValues={DynamicGPUDiscovery,SelfHosted,LogPosting,MultiNodeWorkloads}' +``` + +**Helmfile** + +The Helmfile deployment uses the same `selfManaged.featureGateValues` chart value. By +default, the helmfile does not set this field, so the chart default +`["DynamicGPUDiscovery"]` applies. + +To override, add `featureGateValues` to the worker release values in +`helmfile.d/03-worker.yaml.gotmpl`: + +```yaml +- selfManaged: + featureGateValues: ["DynamicGPUDiscovery", "SelfHosted", "LogPosting"] + imageCredHelper: + imageRepository: {{ .Values.global.image.registry }}/{{ .Values.global.image.repository }}/nvcf-image-credential-helper +``` + +Alternatively, set it in an environment-specific values file (e.g., +`environments/.yaml`) under the same key path, which avoids editing the shared +helmfile template. + +After changing, run `helmfile --selector release-group=workers sync` to apply. + +### Verifying Feature Flags + +After installing or upgrading, verify the active feature flags: + +```bash +nvcf_cluster_name="$(kubectl get nvcfbackends -n nvca-operator -o name | cut -d'/' -f2)" +kubectl get nvcfbackends -n nvca-operator "$nvcf_cluster_name" -o jsonpath='{.spec.featureGate.values}' && echo "" +``` + +The NVCA agent pod command-line args also reflect the active flags: + +```bash +kubectl get pods -n nvca-system -o yaml | grep -i feature +``` + +### Modifying Feature Flags at Runtime + +Feature flags can also be modified at runtime by patching the NVCFBackend resource directly. +This is useful for quick changes without running a `helm upgrade`. + + +Prefer `helm upgrade` with updated values to change feature flags. Direct patches to the +NVCFBackend will be overwritten on the next Helm upgrade. + + + +1. Get the NVCF cluster name: + +```bash +nvcf_cluster_name="$(kubectl get nvcfbackends -n nvca-operator -o name | cut -d'/' -f2)" +``` + +1. View current feature flags: + +```bash +kubectl get nvcfbackends -n nvca-operator -o yaml | grep -A 5 "featureGate:" +``` + +1. Patch the feature flags. Note that this will override all feature flags. + + +When modifying feature flags, you must preserve any existing feature flags you want to keep. The patch command will override all feature flags, so you need to include all desired feature flags in the value array. + + + +```bash +kubectl patch nvcfbackends.nvcf.nvidia.io -n nvca-operator "$nvcf_cluster_name" --type=merge -p '{"spec":{"overrides":{"featureGate":{"values":["LogPosting","CachingSupport"]}}}}' +``` + +As an alternative to the patch command, you can also modify the feature flags using the edit command: + +```bash +kubectl edit nvcfbackend -n nvca-operator +... +spec: + featureGate: + values: + - LogPosting # Existing feature flag + overrides: + featureGate: + values: + - LogPosting # Existing feature flag copied over + - -CachingSupport # Caching support disabled + ... +``` + +1. Verify the changes: + +```bash +kubectl get pods -n nvca-system -o yaml | grep -i feature +``` + +## CPU-only pod scheduling (HelmAllowCPUNodes) + + +Supported in Cluster Agent 2.50.4 or higher. + + + +When `HelmAllowCPUNodes` is enabled, NVCA schedules CPU-only pods from Helm-based functions on +non-GPU nodes while keeping GPU pods on GPU nodes with their required instance-type affinity. +This reduces infrastructure cost and improves GPU utilization. + +**Scheduling behavior** + +| Pod type | Scheduling behavior | +| ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| GPU pods | Required instance-type node affinity (unchanged). | +| CPU-only pods | Preferred anti-affinity (weight 100) for nodes with the instance-type label; schedules on CPU-only nodes when available, can fall back to GPU nodes if needed. | + + +`HelmAllowCPUNodes` cannot be enabled when `HelmResourceConstraints` is enabled. +`HelmResourceConstraints` is enabled by default. You must disable it first by adding +`-HelmResourceConstraints` to the feature gate values, then add `HelmAllowCPUNodes`. + + + +**How to enable** + +1. Follow the steps in [Modifying Feature Flags at Runtime](./configuration.md). + +2. In `spec.overrides.featureGate.values`, include all existing flags you want to keep, add + `-HelmResourceConstraints`, then add `HelmAllowCPUNodes`. + + Example (preserve your existing flags and add the two changes): + + ```yaml + spec: + overrides: + featureGate: + values: + - # copy existing flags here + - -HelmResourceConstraints + - HelmAllowCPUNodes + ``` + +**How to disable** + +Remove `HelmAllowCPUNodes` from the values list, or explicitly disable it: + +```yaml +spec: + overrides: + featureGate: + values: + - -HelmAllowCPUNodes +``` + +## Enable Helm Shared Storage + +The NVIDIA Cluster Agent supports shared storage for Helm charts through the [SMB CSI driver](https://github.com/kubernetes-csi/csi-driver-smb/tree/master/charts#install-csi-driver-with-helm-3). This feature is required for enabling Helm-based tasks in your cluster. + + +The Helm shared storage feature must be enabled before you can use Helm-based tasks in your cluster. This feature provides the necessary storage infrastructure for Helm chart operations. + + + + +When enabling the Helm shared storage feature flag, you must preserve any existing feature flags. The patch command will override all feature flags, so you need to include all desired feature flags in the value array. If you already have other feature flags enabled, you should include them along with "HelmSharedStorage" in the value array. + + + +1. First, install the SMB CSI driver using Helm: + +```bash +helm repo add csi-driver-smb https://raw.githubusercontent.com/kubernetes-csi/csi-driver-smb/master/charts +helm install csi-driver-smb csi-driver-smb/csi-driver-smb --namespace kube-system --version v1.16.0 +``` + +1. Get the NVCF cluster name: + +```bash +nvcf_cluster_name="$(kubectl get nvcfbackends -n nvca-operator -o name | cut -d'/' -f2)" +``` + +1. Enable the Helm shared storage feature flag: + +```bash +kubectl patch nvcfbackends.nvcf.nvidia.io -n nvca-operator "$nvcf_cluster_name" --type=merge -p '{"spec":{"overrides":{"featureGate":{"values":["LogPosting","HelmSharedStorage", "CachingSupport"]}}}}' +``` + +1. Verify that the feature flag is enabled: + +```bash +kubectl get pods -n nvca-system -o yaml | grep HelmSharedStorage +``` + +## Agent Config Merging + +The NVIDIA Cluster Agent supports merging custom configuration into the generated NVCA config +via the `agentConfig.mergeConfig` Helm value. This allows you to override or extend NVCA +runtime settings without modifying the operator's config generation logic. + +When `agentConfig.mergeConfig` is set, the Helm chart creates a ConfigMap called +`agent-config-merge` containing the provided YAML. This ConfigMap is mounted into the NVCA +pod and merged with the generated config at runtime. + +Example values: + +```yaml +agentConfig: + mergeConfig: | + agent: + logLevel: debug +``` + +Local LLM worker transport example: + +```yaml +agentConfig: + mergeConfig: | + workload: + stargateQUICInsecure: true +``` + +`workload.stargateQUICInsecure: true` makes generated LLM workers pass +`--quic-insecure` to the `pylon` sidecar. Use it only for local or +isolated test clusters that run the LLM request router tunnel without TLS. For +the full LLM addon setup, see +[LLM Function Enablement](/nvcf/self-managed/llm-function-enablement). + +BYOO collector debug and log chunking example: + +```yaml +agentConfig: + mergeConfig: | + agent: + byooLogChunking: + enabled: true + maxPayloadBytes: 262144 + byooDebugMode: + enabled: true + byooOtelCollector: + exporterHelper: + timeout: 30s + sendingQueue: + batch: + flushTimeout: 200ms + sizer: bytes + minSize: 1000000 + maxSize: 1000000 + logSampling: + samplingPercentage: 10 + mode: hash_seed + hashSeed: 1234 + failClosed: true + traceSampling: + samplingPercentage: 1 + mode: hash_seed + hashSeed: 1234 + failClosed: true +``` + +`logSampling` and `traceSampling` configure separate probabilistic samplers. +Both support `samplingPercentage`, `mode`, `hashSeed`, and `failClosed`. +`logSampling` also supports `attributeSource`, `fromAttribute`, and +`samplingPriority`. An unset `mode` uses `hash_seed`. Leave either sampling +percentage unset to keep that signal unsampled. + +For `hash_seed`, use `0` or a percentage of at least `0.006103515625`. +`proportional` and `equalizing` log sampling require a TraceID or +`sampling.randomness`; they do not support `attributeSource` or +`fromAttribute`. Set `failClosed: true` to reject log records without usable +randomness. If you configure `samplingPriority`, record values must be greater +than `0`; zero is not supported by the pinned collector version. + +BYOO metric subset example: + +```yaml +agentConfig: + mergeConfig: | + agent: + byooMetricSubset: + enabled: true + filterConfig: | + error_mode: ignore + metric_conditions: + - 'metric.name != "BpsInstrument"' + byooWorkloadMetrics: + dropLabels: + - custom_label +``` + +When `byooMetricSubset.enabled` is true, `dropLabels` extends the default +`metric_subset_enabled` label. The configured labels are removed from both the +primary metrics pipeline and the metric subset endpoint on port `19091`. + +Apply via Helm: + +```bash +helm upgrade nvca-operator -n nvca-operator --create-namespace -i \ + \ + --set-file agentConfig.mergeConfig=my-nvca-config.yaml +``` + +Or include it in a values file passed to `helm upgrade -f values.yaml`. + +## Manual Instance Configuration + + +It is **highly recommended** to rely on Dynamic GPU Discovery (and therefore the NVIDIA GPU Operator), as manual instance configuration is error-prone. + +This type of configuration is only necessary when the cluster cloud provider does not support the NVIDIA GPU Operator. + + + +Manual instance configuration allows you to disable Dynamic GPU Discovery and instead provide a static list of instance types that NVCA will register with the NVCF control plane. This is useful when: + +- You have a known, fixed set of GPU configurations +- Dynamic GPU discovery isn't working correctly for your environment +- You want to control exactly which instance types are available + +By default, NVCA uses Dynamic GPU Discovery to automatically detect GPUs on cluster nodes and register appropriate instance types. When this is disabled, NVCA instead reads a static GPU configuration from a ConfigMap. + +### Prerequisites + +- A working NVCF cluster with `nvca-operator` installed + +- Access to modify Helm values for `nvca-operator` + +- Since you are not using the GPU Operator, you must ensure each GPU node has the instance-type label that matches the "value" field in your manual configuration: + + ```bash + kubectl label nodes nvca.nvcf.nvidia.io/instance-type= + ``` + + For example, if your configuration specifies `"value": "OCI.GPU.A10"`, you would label the node with: + + ```bash + kubectl label nodes gpu-node-1 nvca.nvcf.nvidia.io/instance-type=OCI.GPU.A10 + ``` + +### Step 1: Create the GPU Configuration JSON + +Create a JSON file defining your GPU types and instance configurations. The configuration is an array of GPU types, each containing an array of instance types. + +**Example Configuration (gpu-config.json):** + +```json +[ + { + "name": "H100", + "capacity": 8, + "instanceTypes": [ + { + "name": "ON-PREM.GPU.H100_1x", + "value": "ON-PREM.GPU.H100", + "description": "One Nvidia Hopper GPU", + "default": true, + "cpuCores": 16, + "systemMemory": "128G", + "gpuMemory": "80G", + "gpuCount": 1, + "os": "linux", + "driverVersion": "535.135.05", + "cpuArch": "amd64", + "storage": "1Ti" + }, + { + "name": "ON-PREM.GPU.H100_8x", + "value": "ON-PREM.GPU.H100", + "description": "Eight Nvidia Hopper GPUs (Full Node)", + "default": false, + "cpuCores": 128, + "systemMemory": "1Ti", + "gpuMemory": "640G", + "gpuCount": 8, + "os": "linux", + "driverVersion": "535.135.05", + "cpuArch": "amd64", + "storage": "8Ti" + } + ] + } +] +``` + +### Step 2: Base64 Encode the Configuration + +The GPU configuration must be Base64-encoded for the Helm values. Use the following command: + +```bash +# Base64 encode the configuration (without line wrapping) +GPU_CONFIG_B64=$(cat gpu-config.json | base64 -w 0) +echo $GPU_CONFIG_B64 +``` + + +On macOS, use `base64` without the `-w 0` flag: + +```bash +GPU_CONFIG_B64=$(cat gpu-config.json | base64) +``` + + + +### Step 3: Configure Helm Values + +Update your `nvca-operator` Helm values to disable Dynamic GPU Discovery and provide the manual configuration. + +**Example values.yaml:** + +```yaml +selfManaged: + featureGateValues: [] + gpuManualInstanceConfigB64: "" +``` + + +By default `selfManaged.featureGateValues` is `["DynamicGPUDiscovery"]`. Set it to an empty list (`[]`) to disable dynamic discovery and use your manual configuration instead. + + + +### Step 4: Install or Upgrade the Operator + +Apply the configuration using Helm: + +```bash +helm upgrade nvca-operator -n nvca-operator --create-namespace -i \ + \ + -f values.yaml +``` + +If you are using the NVCF self-hosted Helmfile, add your values file as an entry in the nvca-operator release `values:` list and run `helmfile sync` or `helmfile apply` instead. + +### Configuration Fields Reference + +**GPU Type Fields:** + +| Field | Required | Description | +| --- | --- | --- | +| `name` | Yes | GPU type name (e.g., "A100", "L40", "H100"). Must match the GPU product name reported by `nvidia-smi`. | +| `capacity` | No | Total GPU capacity for this type. Used for resource accounting and quota management. | +| `instanceTypes` | Yes | Array of instance type configurations for this GPU type. | + +**Instance Type Fields:** + +| Field | Required | Description | +| --- | --- | --- | +| `name` | Yes | Unique instance type identifier (e.g., "ON-PREM.GPU.H100_1x"). This is the name users select when deploying functions. | +| `value` | Yes | Instance type value used for internal matching (e.g., "ON-PREM.GPU.H100"). Must match the `nvca.nvcf.nvidia.io/instance-type` node label. | +| `description` | No | Human-readable description displayed in the UI. | +| `default` | No | Whether this is the default instance type for this GPU. Only one instance type per GPU should be marked as default. | +| `cpuCores` | Yes | Number of CPU cores allocated to workloads using this instance type. | +| `systemMemory` | Yes | System RAM allocation (e.g., "28G", "128G", "1Ti"). Uses Kubernetes quantity format. | +| `gpuMemory` | Yes | Total GPU memory for this instance type. For multi-GPU instances, this is the total across all GPUs. | +| `gpuCount` | Yes | Number of GPUs in this instance type. | +| `os` | No | Operating system (e.g., "linux"). | +| `driverVersion` | No | NVIDIA driver version (e.g., "535.135.05"). | +| `cpuArch` | No | CPU architecture (e.g., "amd64", "arm64"). | +| `storage` | No | Storage allocation per instance (e.g., "512Gi", "1Ti"). Uses Kubernetes quantity format. | + +### Verification + +After applying the configuration, verify that NVCA is using the static configuration: + +1. **Check the NVCFBackend resource:** + + ```bash + kubectl get nvcfbackend -n nvca-operator -o yaml + ``` + + Look for `-DynamicGPUDiscovery` in the feature gates and verify the GPU configuration is present. + +2. **Check the nvca-config ConfigMap:** + + ```bash + kubectl get configmap nvca-config -n nvca-system -o yaml + ``` + + The `gpus` key should contain your JSON configuration. + +3. **Check NVCA logs for registration:** + + ```bash + kubectl logs -n nvca-system -l app=nvca | grep -i "registration\|instance" + ``` + +### Troubleshooting + +**Configuration Not Applied:** + +1. Verify Dynamic GPU Discovery is disabled in the feature gate values +2. Ensure the Base64 encoding is correct and doesn't contain line breaks +3. Check that the JSON is valid before encoding + +**Invalid JSON Format:** + +1. Validate your JSON using a JSON validator before encoding +2. Ensure all required fields are present +3. Check that numeric values (`cpuCores`, `gpuCount`) are not quoted as strings + +**Memory/Storage Format Errors:** + +Memory and storage values must use valid Kubernetes quantity format: + +- Valid: `"28G"`, `"128Gi"`, `"1Ti"`, `"512Mi"` +- Invalid: `"28GB"`, `"128 Gi"`, `"1TB"` + + +Use `G` or `Gi` for gigabytes, `T` or `Ti` for terabytes. The `i` suffix indicates binary units (1024-based). + + + +## Cloud Provider-Specific Notes + +## Oracle Cloud Infrastructure (OCI) + +When using Oracle Container Engine for Kubernetes (OKE), ensure that: + +- Your compute nodes and GPU nodes are in the same availability domain +- This is required for proper network connectivity between the NVIDIA Cluster Agent and GPU nodes +- Flannel CNI is the current recommended and validated CNI vs OCI native CNI for OKE cluster networking. diff --git a/docs/user/cluster-management/container-cache.md b/docs/compute-plane/cluster-management/container-cache.md similarity index 100% rename from docs/user/cluster-management/container-cache.md rename to docs/compute-plane/cluster-management/container-cache.md diff --git a/docs/user/cluster-management/gang-scheduling.md b/docs/compute-plane/cluster-management/gang-scheduling.md similarity index 100% rename from docs/user/cluster-management/gang-scheduling.md rename to docs/compute-plane/cluster-management/gang-scheduling.md diff --git a/docs/user/cluster-management/gxcache.md b/docs/compute-plane/cluster-management/gxcache.md similarity index 100% rename from docs/user/cluster-management/gxcache.md rename to docs/compute-plane/cluster-management/gxcache.md diff --git a/docs/compute-plane/cluster-management/index.md b/docs/compute-plane/cluster-management/index.md new file mode 100644 index 0000000000..54d9037214 --- /dev/null +++ b/docs/compute-plane/cluster-management/index.md @@ -0,0 +1,48 @@ +# GPU Cluster Setup + +The NVIDIA Cluster Agent (NVCA) connects GPU clusters to the NVCF control plane, enabling them to act as deployment targets for Cloud Functions. NVCA is a function deployment orchestrator that registers a cluster's GPU resources, communicates with the control plane, and manages the lifecycle of function deployments on GPU nodes. + +For a fresh install, use the [Quickstart](/nvcf/overview/quickstart). The one-click CLI flow can register a GPU cluster as part of the install. Use this section for manual cluster registration, standalone NVCA installation, and day-two cluster configuration. + + +If you pin NVCA separately from the recommended compute-plane stack, check the +current NVCA version before upgrading to NVCA 3.x. Clusters running NVCA 2.51.0 +or earlier have version-specific upgrade guidance. See the +[0.6.0 upgrade notes](https://docs.nvidia.com/nvcf/v0.6.0/0-6-0-release-notes#upgrade-notes). + + +After installing NVCA on a cluster: + +- The registered cluster will show as a deployment option in the `GET /v2/nvcf/clusterGroups` API response. +- Any functions under the cluster's authorized NCA IDs can now deploy on the cluster. + +## Authentication and Keys + +| Key Type | Description | +| --- | --- | +| NVCF API Key (NAK) | Used by NVCA to authenticate with the control plane. See [self-hosted-api](/nvcf/self-managed/api) for details on API key generation. | + +## Prerequisites + +- Access to a Kubernetes cluster including GPU-enabled nodes ("GPU cluster") + + - The cluster must have a compatible version of [Kubernetes](https://kubernetes.io/releases/). + + - The cluster must have the [NVIDIA GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html#operator-install-guide) installed. + + - If your cloud provider does not support the NVIDIA GPU Operator, [Manual Instance Configuration](./configuration.md) is possible, but not recommended due to lack of maintainability. + - To get the most out of clusters with multi-node NVLink ([MNNVL](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/dra-cds.html#dra-docs-compute-domains)) GPUs like [GB200](https://www.nvidia.com/en-us/data-center/gb200-nvl72/), the [NVIDIA GPU DRA driver](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/dra-intro-install.html) must be installed. See the [nvlink-optimized-clusters](./configuration.md) for details. + - For development or testing environments without physical GPUs, install the [fake-gpu-operator](../fake-gpu-operator) instead. + +- Registering the cluster requires `kubectl` and `helm` installed. + +- The user registering the cluster must have the `cluster-admin` role privileges to install the NVIDIA Cluster Agent Operator (`nvca-operator`). + +### Supported Kubernetes Versions + +- Supported versions are the latest Kubernetes minor release and the two prior minor releases (N-2). See official Kubernetes docs for current supported [versions](https://kubernetes.io/releases/version-skew-policy/#supported-versions). + +### Considerations + +- The NVIDIA Cluster Agent currently only supports caching if the cluster is enabled with `StorageClass` configurations. If the "Caching Support" capability is enabled, the agent will make the best effort by attempting to detect storage during deployments and fall back on non-cached workflows. +- Each function and task requires several infrastructure containers be deployed alongside workload containers. These infrastructure containers collectively need 6 CPU cores and 8 Gi of system memory to execute. Each GPU node must have at least this many resources, ideally significantly more for workload resource usage. diff --git a/docs/user/cluster-management/kai-scheduler.md b/docs/compute-plane/cluster-management/kai-scheduler.md similarity index 100% rename from docs/user/cluster-management/kai-scheduler.md rename to docs/compute-plane/cluster-management/kai-scheduler.md diff --git a/docs/user/cluster-management/model-cache.md b/docs/compute-plane/cluster-management/model-cache.md similarity index 100% rename from docs/user/cluster-management/model-cache.md rename to docs/compute-plane/cluster-management/model-cache.md diff --git a/docs/user/cluster-management/monitoring.md b/docs/compute-plane/cluster-management/monitoring.md similarity index 100% rename from docs/user/cluster-management/monitoring.md rename to docs/compute-plane/cluster-management/monitoring.md diff --git a/docs/compute-plane/cluster-management/nsight-profiling.md b/docs/compute-plane/cluster-management/nsight-profiling.md new file mode 100644 index 0000000000..f798a97a7d --- /dev/null +++ b/docs/compute-plane/cluster-management/nsight-profiling.md @@ -0,0 +1,397 @@ +# Nsight Profiling + +NVIDIA Nsight Operator can inject NVIDIA Nsight Systems profiling support into +NVCF function pods that run on a self-hosted GPU cluster. Use it after the NVCA +Operator is installed and the cluster can deploy and invoke functions. + +Nsight Operator injects profiling only into newly admitted pods. To enable +profiling for NVCF functions, label the workload namespace before creating or +recreating the function pods. + +## Prerequisites + +- A self-hosted NVCF control plane with a registered GPU cluster. +- A healthy NVCA Operator and NVCA agent. See [Self-Managed Clusters](./self-managed.md). +- The NVIDIA GPU Operator installed on the GPU cluster. +- `kubectl` and Helm access with permissions to install cluster-scoped resources. +- An external S3-compatible bucket for Nsight profiling results. +- The NVIDIA Nsight Operator resources bundle, including `nsight_operator.py`, + installed on the workstation that will control profiling sessions. + +Install the Python dependencies from the unpacked Nsight Operator resources +bundle: + +```bash +python3 -m pip install -r requirements.txt +``` + +## Storage + +Use external S3-compatible storage for repeatable NVCF profiling runs. The +stock Nsight Operator chart can deploy an in-cluster MinIO instance, but that +default is intended for short-lived experiments. External object storage keeps +captures available after Kubernetes pod restarts and makes downloads +independent of the cluster lifecycle. + +Create a Kubernetes Secret in the Nsight Operator namespace. Replace the +placeholder values with credentials for your storage provider. + +```yaml title="nsight-s3-secret.yaml" +apiVersion: v1 +kind: Secret +metadata: + name: nsight-s3-credentials + namespace: nsight-operator +type: Opaque +stringData: + storage-config.yaml: | + storage_type: s3 + bucket_name: + aws_access_key_id: + aws_secret_access_key: + region_name: + endpoint_url: + local_cache_dir: /tmp/nsight-s3-cache +``` + +Apply the namespace and Secret: + +```bash +kubectl create namespace nsight-operator --dry-run=client -o yaml | kubectl apply -f - +kubectl apply -f nsight-s3-secret.yaml +``` + +If your storage provider does not require `endpoint_url`, omit it from +`storage-config.yaml`. + +## Install Nsight Operator + +The profiling-only profile enables coordinator mode for on-demand captures, +uses external S3 storage, disables operator-managed MinIO, and disables OTLP +trace mirroring. + +```yaml title="nsight-values-profile-only.yaml" +cloudStorage: + enabled: true + minio: + enabled: false + secretRef: + name: nsight-s3-credentials + +nsight-injector: + nsightToolConfig: + coordinator: true + otlpMirroringEnabled: false + nsightToolArgs: "-t cuda,nvtx,osrt --python-sampling=true --cuda-graph-trace=node" + +nsight-otel-collector: + enabled: false + +otlpProxyConfig: + enabled: false +``` + +Install the chart: + +```bash +helm upgrade --install nsight-operator \ + --wait \ + --namespace nsight-operator \ + --create-namespace \ + --values nsight-values-profile-only.yaml \ + https://helm.ngc.nvidia.com/nvidia/devtools/charts/nsight-operator-26.2.2.tgz +``` + +To also enable OTLP trace mirroring, use this values file instead: + +```yaml title="nsight-values-with-otlp.yaml" +cloudStorage: + enabled: true + minio: + enabled: false + secretRef: + name: nsight-s3-credentials + +nsight-injector: + nsightToolConfig: + coordinator: true + otlpMirroringEnabled: true + nsightToolArgs: "-t cuda,nvtx,osrt --python-sampling=true --cuda-graph-trace=node" + +nsight-otel-collector: + enabled: true + +otlpProxyConfig: + enabled: true +``` + +Verify that the Nsight Operator components are running: + +```bash +kubectl get pods -n nsight-operator +kubectl get nsightcloudstorageconfigs -A +kubectl get nsightgateways -A +kubectl get nsightcoordinators -A +``` + +## Enable Profiling Manually + +Container functions run in the shared `nvcf-backend` namespace. Label that +namespace before deploying or recreating container function pods: + +```bash +kubectl label namespace nvcf-backend \ + nvidia-nsight-profile=enabled \ + --overwrite +``` + +Helm functions run in dedicated namespaces created by NVCA. After deploying a +Helm function, find the function namespace and label it: + +```bash +kubectl get namespaces \ + -l nvca.nvcf.nvidia.io/workload-instance-type=miniservice + +kubectl label namespace \ + nvidia-nsight-profile=enabled \ + --overwrite +``` + +Existing pods are not injected retroactively. Recreate function pods after +adding the label. For container functions, delete the affected pod and allow +NVCA to recreate it: + +```bash +kubectl get pods -n nvcf-backend +kubectl delete pod -n nvcf-backend +``` + +For Helm functions, redeploy the function or restart the workload resource in +the function namespace: + +```bash +kubectl get deploy,statefulset,pod -n +kubectl rollout restart deployment/ -n +``` + +## Optional Kyverno Automation + +Kyverno can label new NVCF workload namespaces as they are created. This covers +container functions through `nvcf-backend` when the namespace is created after +the policy exists, and Helm functions through their dedicated NVCA-created +namespaces. + +Install Kyverno before applying this policy. Then create the ClusterPolicy: + +```yaml title="nsight-label-nvcf-namespaces.yaml" +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: label-nvcf-namespaces-for-nsight +spec: + background: true + rules: + - name: add-nsight-profile-label + match: + any: + - resources: + kinds: + - Namespace + selector: + matchExpressions: + - key: nvca.nvcf.nvidia.io/workload-instance-type + operator: In + values: + - pod_spec + - miniservice + mutate: + patchStrategicMerge: + metadata: + labels: + nvidia-nsight-profile: enabled +``` + +Apply the policy: + +```bash +kubectl apply -f nsight-label-nvcf-namespaces.yaml +``` + +Verify namespace labels: + +```bash +kubectl get namespace nvcf-backend --show-labels +kubectl get namespaces \ + -l nvca.nvcf.nvidia.io/workload-instance-type \ + --show-labels +``` + +If `nvcf-backend` or existing Helm function namespaces were created before the +policy, label them manually or use Kyverno mutate-existing features in your +cluster policy. Recreate existing function pods after the label is present. + +## Run a Capture + +Use this flow with an existing Gemma-based LLM function, or create one using the +[LLM Gateway](/nvcf/self-managed/llm-gateway#function-configuration) function configuration +pattern. Keep the model name aligned with the function's configured +`models[].name` value. + +Set the function and invocation variables: + +```bash +export GATEWAY_ADDR= +export FUNCTION_ID= +export FUNCTION_VERSION_ID= +export NVCF_TOKEN= +export NVCF_INVOKE_KEY= +export NVCF_BACKEND= +export NVCF_GPU= +export NVCF_INSTANCE_TYPE= +export GEMMA_MODEL_NAME= +``` + +Deploy the function with a single GPU if it is not already deployed: + +```bash +curl -s -X POST \ + "http://${GATEWAY_ADDR}/v2/nvcf/deployments/functions/${FUNCTION_ID}/versions/${FUNCTION_VERSION_ID}" \ + -H "Host: api.${GATEWAY_ADDR}" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${NVCF_TOKEN}" \ + -d '{ + "deploymentSpecifications": [ + { + "backend": "'"${NVCF_BACKEND}"'", + "gpu": "'"${NVCF_GPU}"'", + "instanceType": "'"${NVCF_INSTANCE_TYPE}"'", + "minInstances": 1, + "maxInstances": 1 + } + ] + }' | jq . +``` + +Wait until the function pod is running and has been recreated after the Nsight +label was applied: + +```bash +kubectl get pods -n nvcf-backend +kubectl get pods -A -l function-id="${FUNCTION_ID}" +``` + +Configure the Nsight CLI. `autoconfigure` discovers the Nsight Gateway and +storage settings from the cluster. + +```bash +python3 nsight_operator.py autoconfigure -n nsight-operator +``` + +Start a profiling session: + +```bash +python3 nsight_operator.py session-begin --title gemma-nvcf-smoke +python3 nsight_operator.py profiler-start +``` + +Invoke the model through the NVCF LLM route while profiling is active: + +```bash +for i in 1 2 3; do + curl -s -X POST "http://${GATEWAY_ADDR}/v1/chat/completions" \ + -H "Host: llm.invocation.${GATEWAY_ADDR}" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${NVCF_INVOKE_KEY}" \ + -d '{ + "model": "'"${FUNCTION_ID}/${GEMMA_MODEL_NAME}"'", + "messages": [ + { + "role": "user", + "content": "Write a short paragraph about GPU profiling." + } + ], + "max_tokens": 128, + "temperature": 0.2 + }' + echo +done +``` + +Stop and close the session: + +```bash +python3 nsight_operator.py profiler-stop +python3 nsight_operator.py session-end +``` + +List and download the report: + +```bash +python3 nsight_operator.py ls +python3 nsight_operator.py download --output-dir ./nsight-profiles +``` + +The downloaded directory should contain one or more `.nsys-rep` files. Open the +reports with NVIDIA Nsight Systems. + +## Troubleshooting + +### Pods are not injected + +Check the namespace label: + +```bash +kubectl get namespace --show-labels +``` + +The namespace must have `nvidia-nsight-profile=enabled` before the pod is +created. If the pod already existed, recreate it. Also check the Nsight Operator +logs and verify that the process name matches the configured +`injectionIncludePatterns`. + +```bash +kubectl logs -n nsight-operator -l app.kubernetes.io/name=nsight-operator --tail 100 +kubectl describe pod -n +``` + +### Downloads fail + +Verify that the storage Secret exists and that the bucket credentials can write +and read profiling reports: + +```bash +kubectl get secret nsight-s3-credentials -n nsight-operator +kubectl get nsightcloudstorageconfigs -n nsight-operator -o yaml +``` + +If `nsight_operator.py configure` was used instead of `autoconfigure`, configure +storage access separately before running `download`. + +### The Nsight Gateway is not reachable + +Check the Nsight Gateway service: + +```bash +kubectl get svc -n nsight-operator +kubectl get nsightgateways -n nsight-operator -o yaml +``` + +For a ClusterIP service, run `nsight_operator.py autoconfigure -n nsight-operator` +from a workstation with `kubectl` access. The CLI can set up port forwarding for +the gateway. + +### GPU metrics collectors conflict + +Some GPU profiling and metrics collectors use the same low-level GPU interfaces. +If profiling fails after injection, check whether another DCGM or GPU metrics +collector is running on the node. Temporarily disable the conflicting collector +or profile in a maintenance window. + +## See Also + +- [Self-Managed Clusters](./self-managed.md) +- [LLM Gateway](/nvcf/self-managed/llm-gateway) +- [Generic HTTP Function Invocation](/nvcf/self-managed/generic-http-function-invocation) +- [NVIDIA Nsight Operator Installation Guide](https://docs.nvidia.com/nsight-operator/InstallationGuide/index.html) +- [NVIDIA Nsight Operator User Guide](https://docs.nvidia.com/nsight-operator/UserGuide/index.html) +- [NVIDIA Nsight Operator CRD Reference](https://docs.nvidia.com/nsight-operator/CRDReference/index.html) diff --git a/docs/user/cluster-management/reference.md b/docs/compute-plane/cluster-management/reference.md similarity index 100% rename from docs/user/cluster-management/reference.md rename to docs/compute-plane/cluster-management/reference.md diff --git a/docs/compute-plane/cluster-management/self-managed.md b/docs/compute-plane/cluster-management/self-managed.md new file mode 100644 index 0000000000..728098a6ed --- /dev/null +++ b/docs/compute-plane/cluster-management/self-managed.md @@ -0,0 +1,772 @@ +# Self-Managed Clusters + +GPU clusters are registered with the NVCF control plane and managed by the NVCA +Operator. Use the compute-plane Makefile in +`deploy/stacks/nvcf-compute-plane` for this workflow. It registers one GPU +cluster at a time with `nvcf-cli`, then installs the operator with the cluster +identity returned by registration. The operator reads that identity from a +local ConfigMap and authenticates through the local OpenBao (Vault) instance. + + +A running NVCF control plane (SIS, OpenBao, NATS, Cassandra, and all core +services) is required. The [Quickstart](/nvcf/overview/quickstart) can install the +control plane and register a GPU cluster in one flow. Use this page when you +need to install or operate the NVCA Operator after using the Helmfile +installation path. + + + +Clone the public repository and run the compute-plane commands from its root: + +```bash +git clone https://github.com/nvidia/nvcf.git +cd nvcf +``` + +## Prerequisites + +Before installing the NVCA Operator, ensure the following prerequisites are met: + +- The [control plane](/nvcf/self-managed/helmfile-installation) is installed and all core services are running. + +- The [NVIDIA GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) is installed on the GPU cluster. The GPU Operator manages the NVIDIA drivers, device plugin, and GPU feature discovery required for workload scheduling. For development or testing environments without physical GPUs, see [fake-gpu-operator](../fake-gpu-operator). + +- (Optional) Install [KAI Scheduler](./kai-scheduler.md) for GPU bin-packing + and queues. KAI is also the scheduling foundation for + [gang scheduling](./gang-scheduling.md) and + [topology-aware scheduling](./topology-aware-scheduling.md) with Grove and + Dynamo. + +- `GPU Workload Components` must be available in a user-managed registry that your Kubernetes cluster can access. See `GPU Workload Components` under [self-hosted-artifact-manifest](/nvcf/overview/manifest) for necessary artifacts and [self-hosted-image-mirroring](/nvcf/overview/image-mirroring) for mirroring instructions. + +- `nvcf-cli` is available on the deployment machine. The compute-plane Makefile + calls it during cluster registration. See [self-hosted-cli](/nvcf/self-managed/cli) for + CLI installation and configuration. + +- The [SMB CSI driver](https://github.com/kubernetes-csi/csi-driver-smb) (`smb.csi.k8s.io`) must be installed on the GPU cluster. It is required for NVCA shared model cache storage (samba sidecar). Install it with: + + ```bash + helm repo add csi-driver-smb \ + https://raw.githubusercontent.com/kubernetes-csi/csi-driver-smb/master/charts + + helm install csi-driver-smb csi-driver-smb/csi-driver-smb \ + -n kube-system --version v1.17.0 + ``` + +## How It Works + +In self-managed mode, the cluster identity comes from the CLI registration +step, and the operator consumes it: + +1. You configure the compute-plane Helmfile environment with artifact registry + settings and deployment-specific overrides. +2. You export the control-plane profile. The profile is the canonical handoff + for the control-plane identity, reachable endpoints, host overrides, and + transport trust. +3. From the repository root, you run + `make -C deploy/stacks/nvcf-compute-plane register-cluster` (see + [Register the cluster](#register-the-cluster)). The target runs + `nvcf-cli self-hosted compute-plane register` with the profile. The CLI + discovers the GPU cluster's OIDC issuer and public JWKS, records them with + the control plane (ICMS), and writes the `clusterID`, `clusterGroupID`, + identity source, endpoints, host overrides, and transport trust to a + registration values file. +4. From the repository root, you run + `make -C deploy/stacks/nvcf-compute-plane install`. Helmfile installs + `helm-nvca-operator` with the compute-plane environment and registration + values for that cluster, using the same selected kubeconfig context as + registration when `COMPUTE_KUBE_CONTEXT` is set. +5. The Helm chart renders a local ConfigMap (`nvcfbackend-self-managed`) holding the cluster + identity and the SIS, ReVal, and NATS endpoints (plus any host-header overrides). The + operator reads that ConfigMap and creates the NVCFBackend resource. +6. The operator creates the NVCA agent pod. The agent authenticates to the control plane + with a projected service account token (PSAT), which the control plane validates against + the issuer and JWKS registered in step 3, and begins managing GPU workloads. +7. Runtime secrets are injected by the OpenBao vault-agent sidecar, which + authenticates using Kubernetes service account JWT tokens against the local + OpenBao instance. Kubernetes image pull secrets are configured separately in + the compute-plane environment and namespaces. + +```mermaid +sequenceDiagram + actor Operator + participant CP as Control-plane stack + participant Profile as Control-plane profile + participant CLI as nvcf-cli + participant Compute as Compute cluster + participant Values as Registration values + participant Helmfile + Operator->>CLI: profile export + CLI->>CP: read selected environment + CLI->>Profile: write generated profile + Operator->>CLI: init with selected config path + Operator->>CLI: compute-plane register + CLI->>Profile: read endpoints and trust + CLI->>Compute: discover OIDC issuer and JWKS + CLI->>CP: register cluster identity + CLI->>Values: write generated NVCA values + Operator->>Helmfile: make install + Helmfile->>Values: consume generated values + Helmfile->>Compute: install NVCA operator +``` + +## Configure the compute plane + +Create the environment file from the repository root. It provides the chart +registry, image registry, image pull secret name, and deployment-specific +overrides used by the NVCA operator. The generated registration values provide +the control-plane service URLs and host overrides. + +```bash +cd path/to/nvcf +touch deploy/stacks/nvcf-compute-plane/environments/.yaml +``` + +```yaml title="deploy/stacks/nvcf-compute-plane/environments/.yaml" +global: + helm: + sources: + registry: + repository: + + image: + registry: + repository: + + imagePullSecrets: + - name: nvcr-pull-secret +``` + +The profile exporter derives the service URLs and route-matching host overrides +from the selected control-plane environment, and registration writes them as +the default values for the compute-plane install. Non-empty +`global.nvcaOperator.selfManaged.icmsServiceURL`, +`icmsServiceHostHeaderOverride`, `revalServiceURL`, +`revalServiceHostHeaderOverride`, `natsURL`, and `natsHostOverride` fields in +the selected compute-plane environment take precedence. Verify the effective +compute-reachable endpoints from those inputs resolve from the GPU cluster. See +[gateway-routing](/nvcf/self-managed/gateway-routing) for service DNS and TLS guidance. + +## Register the cluster + +Register the GPU cluster with the control plane before installing the operator. +The `nvcf-cli` discovers the cluster's OIDC issuer and JWKS and records them +with the control plane, then returns the Helm values the operator needs. See +[self-hosted-cli](/nvcf/self-managed/cli) for CLI installation and configuration, and the +[Cluster Registration](/nvcf/self-managed/cli#cluster-registration) reference for full flag +and output details. + + +The control plane must expose its issuer for the CLI to discover. The Helmfile +installation path enables this with +`openbao.migrations.issuerDiscovery.enabled: true`. + + + +Export the profile from the selected control-plane Helmfile environment. The +default output is +`deploy/stacks/self-managed/out/control-plane-profile.yaml`: + +```bash +nvcf-cli self-hosted \ + --control-plane-stack deploy/stacks/self-managed \ + --env \ + control-plane profile export + +nvcf-cli --config init +``` + +When the LLM add-on is disabled, the generated profile and registration values +omit its optional router and PKI configuration. Enabling LLM requires the +control-plane environment to provide the corresponding managed PKI settings. + +Run registration against the GPU cluster kubeconfig. This is required for +multi-cluster installs because the CLI discovers the OIDC issuer and JWKS from +the target Kubernetes cluster. Set `COMPUTE_KUBE_CONTEXT` to select the GPU +cluster explicitly when the kubeconfig contains multiple contexts. + +```bash +make -C deploy/stacks/nvcf-compute-plane register-cluster \ + CLUSTER_NAME= \ + CLUSTER_REGION= \ + CONTROL_PLANE_PROFILE="$(pwd)/deploy/stacks/self-managed/out/control-plane-profile.yaml" \ + KUBECONFIG_FILE= \ + COMPUTE_KUBE_CONTEXT= \ + NVCF_CLI= \ + NVCF_CLI_CONFIG= +``` + +`NVCF_CLI_CONFIG` is optional when the default CLI config is correct. The Make +target forwards a configured path to registration but does not run `init`. + +The target writes +`registration/-register-values.yaml`. It carries the cluster +identity and endpoints: + +```yaml +clusterID: +clusterGroupID: +ncaID: +selfManaged: + region: + identitySource: psat + icmsServiceURL: "http://" + revalServiceURL: "http://" + natsURL: "nats://:4222" +``` + +`selfManaged.identitySource` is retained for CLI teardown and is not consumed +by the NVCA Operator chart. + +The `template`, `install`, and `apply` targets copy this file into `out/` before +running Helmfile. + +## Installing the NVCA Operator + +| Chart | `helm-nvca-operator` | +| --- | --- | +| Version | `1.28.0` | +| Namespace | `nvca-operator` | +| Depends on | All control-plane services and gateway must be running | + +The compute-plane Helmfile passes these values to the chart: + +| Value | Source | +| --- | --- | +| Cluster identity | `registration/-register-values.yaml` | +| Operator, agent, image credential helper, and shared storage image repositories | `global.image.*` in `environments/.yaml` | +| Control-plane service URLs and host-header overrides | Generated registration values by default; non-empty `global.nvcaOperator.selfManaged.*` endpoint fields in `environments/.yaml` take precedence | +| Image pull secrets | `global.imagePullSecrets` | + +Use `global.nvcaOperator.nodeSelector`, `global.nvcaOperator.tolerations`, and +`global.nvcaOperator.agent.*` in the compute-plane environment when you need to +place the operator, agent, or workloads on specific nodes. + + +The `KAIScheduler` feature flag is optional. Enable it only if the +[KAI Scheduler](./kai-scheduler.md) is installed on the GPU cluster. The flag has no effect, +and GPU workload scheduling will not work as expected, if KAI Scheduler is not present. Omit +`KAIScheduler` from `featureGateValues` when KAI Scheduler is not installed. + + + + +For the full list of available feature flags and how to set or modify them, see +[managing-feature-flags](./configuration.md). + + + +### Node inotify limits + +The NVCA operator and agent use file watchers for ConfigMap and Secret reconciliation. +Some node images set `fs.inotify.max_user_instances` to `128`, which can be too low for +nodes running the full NVCF stack. When the node exhausts inotify instances, NVCA logs +errors such as `failed to create watcher` and `too many open files`. Function creation +or deployment requests that wait for NVCA reconciliation can then return HTTP 500 or +time out. + +Before installing the operator, set higher inotify limits on every node. The following +DaemonSet sets the values on current nodes and on nodes added later: + +```yaml title="inotify-tuner.yaml" +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-inotify-tuner + namespace: kube-system +spec: + selector: + matchLabels: + app.kubernetes.io/name: node-inotify-tuner + template: + metadata: + labels: + app.kubernetes.io/name: node-inotify-tuner + spec: + hostPID: true + tolerations: + - operator: Exists + priorityClassName: system-node-critical + terminationGracePeriodSeconds: 1 + initContainers: + - name: set-sysctl + image: busybox:1.36 + securityContext: + privileged: true + command: + - sh + - -c + - | + set -e + mkdir -p /host/etc/sysctl.d + { + echo 'fs.inotify.max_user_instances=8192' + echo 'fs.inotify.max_user_watches=524288' + } > /host/etc/sysctl.d/99-inotify.conf + nsenter -t 1 -m -u -i -n -p -- sysctl -w fs.inotify.max_user_instances=8192 + nsenter -t 1 -m -u -i -n -p -- sysctl -w fs.inotify.max_user_watches=524288 + volumeMounts: + - name: host + mountPath: /host + containers: + - name: pause + image: registry.k8s.io/pause:3.9 + resources: + requests: + cpu: "1m" + memory: "1Mi" + limits: + cpu: "10m" + memory: "16Mi" + volumes: + - name: host + hostPath: + path: / +``` + +Apply the DaemonSet and wait for it to run on every node: + +```bash +kubectl apply -f inotify-tuner.yaml +kubectl -n kube-system rollout status ds/node-inotify-tuner --timeout=5m +``` + +If your cluster cannot pull public images, mirror `busybox:1.36` and +`registry.k8s.io/pause:3.9` to your registry and update the image fields before applying +the DaemonSet. + +### Image Pull Secrets + +The NVCA operator, NVCA agent, samba sidecar, and image-credential-helper all pull +container images from the registry configured in your compute-plane environment +file. If that registry is private, create the Kubernetes pull secret before +installing the operator. + + +Use a pre-existing pull secret through `global.imagePullSecrets`. Do not use +`generateImagePullSecret`; it does not work in self-managed mode. + + + +Create the secret in the GPU cluster namespaces used by the operator and its +managed resources: + +```bash +for ns in nvca-operator nvca-system nvcf-backend; do + kubectl --kubeconfig \ + create namespace "$ns" --dry-run=client -o yaml | kubectl --kubeconfig apply -f - + kubectl --kubeconfig \ + create secret docker-registry nvcr-pull-secret \ + --docker-server=${REGISTRY} \ + --docker-username='$oauthtoken' \ + --docker-password="$REGISTRY_PASSWORD" \ + --namespace="$ns" \ + --dry-run=client -o yaml | kubectl --kubeconfig apply -f - +done +``` + +Replace `${REGISTRY}` with your container registry (e.g., `nvcr.io`). For non-NGC +registries, replace `--docker-username` and `--docker-password` with your registry +credentials. For NGC (`nvcr.io`), `$REGISTRY_PASSWORD` is your NGC Personal Key or +API Key. + +Reference the secret in your compute-plane environment: + +```yaml +global: + imagePullSecrets: + - name: nvcr-pull-secret +``` + +Helmfile passes this value to the operator chart. The operator adds the pull +secret reference to the pods it manages. Pre-creating the secret in +`nvca-system` and `nvcf-backend` prevents startup races for operator-managed +resources that pull private images. + +### Install + +Install with the compute-plane Makefile. The command copies +`registration/-register-values.yaml` into `out/`, then runs +Helmfile against the GPU cluster kubeconfig: + +```bash +make -C deploy/stacks/nvcf-compute-plane install \ + CLUSTER_NAME= \ + HELMFILE_ENV= \ + NCA_ID= \ + KUBECONFIG_FILE= \ + COMPUTE_KUBE_CONTEXT= +``` + +During installation, Helmfile will: + +1. Create the operator deployment with vault agent annotations for OpenBao auth. +2. Render the `nvcfbackend-self-managed` ConfigMap from the register values (cluster + identity, SIS/ReVal/NATS endpoints, and any host-header overrides). +3. Start the operator, which reads the ConfigMap and creates the NVCFBackend and NVCA agent + deployment. + +### Verify + +Check the operator pod is running. The pod runs the operator, the `nvca-mirror` sidecar, and +the OpenBao vault-agent sidecar (and a `cluster-validator` init container when network +validation is enabled): + +```bash +kubectl get pods -n nvca-operator + +# Expected: +# NAME READY STATUS RESTARTS AGE +# nvca-operator-... Running 0 1m +``` + +Check the cluster identity ConfigMap was rendered from the register values: + +```bash +kubectl get cm nvcfbackend-self-managed -n nvca-operator \ + -o jsonpath='{.data.cluster-dto\.yaml}' + +# Expected: cluster-dto.yaml with non-empty clusterId and clusterGroupId +``` + +Check the NVCFBackend resource was created: + +```bash +kubectl get nvcfbackends -n nvca-operator + +# Expected: one NVCFBackend resource with version and health status +``` + +Check the NVCA agent pod is running (the operator creates this automatically): + +```bash +kubectl get pods -n nvca-system + +# Expected: +# NAME READY STATUS RESTARTS AGE +# nvca-... 3/3 Running 0 2m +``` + + +The NVCA agent pod has 3 containers: the agent, the admission webhook, and the OpenBao vault agent sidecar. +Both should show `Running`. If the vault agent sidecar is in `CrashLoopBackOff`, verify +that OpenBao is healthy and the migration jobs completed successfully. + + + +Verify GPU discovery: + +```bash +kubectl get nvcfbackends -n nvca-operator -o jsonpath='{.items[0].status}' | python3 -m json.tool + +# Look for GPU information in the status output +``` + +### Optional Nsight Profiling + +After NVCA is healthy, you can install NVIDIA Nsight Operator on the GPU cluster +to collect NVIDIA Nsight Systems reports from function pods. See +[Nsight Profiling](./nsight-profiling.md) for external S3 storage setup, +namespace labeling, Kyverno automation, and capture commands. + +### Verify Workload Scheduling + +Use this check only when worker pods on the GPU cluster can reach the +control-plane worker endpoints used by NVCF. For multi-cluster EKS installs, +first validate the GPU cluster by checking `nvca-operator` rollout and +`NVCFBackend` agent health. Function deployment and invocation are valid +acceptance checks only after the worker endpoints are reachable from the GPU +cluster. + +1. Set up environment variables: + +```bash +# Get the Gateway address (from Step 1) +export GATEWAY_ADDR=$(kubectl get gateway nvcf-gateway -n envoy-gateway -o jsonpath='{.status.addresses[0].value}') +echo "Gateway Address: $GATEWAY_ADDR" +``` + +1. Generate an admin token: + +```bash +# Generate an admin API token +export NVCF_TOKEN=$(curl -s -X POST "http://${GATEWAY_ADDR}/v1/admin/keys" \ + -H "Host: api-keys.${GATEWAY_ADDR}" \ + | grep -o '"value":"[^"]*"' | cut -d'"' -f4) + +echo "Token generated: ${NVCF_TOKEN:0:20}..." +``` + +1. Create, deploy, and invoke a test function: + +```bash +# Create a test function +# Replace / with your container registry +# This should match the registry you set in the secrets file +curl -s -X POST "http://${GATEWAY_ADDR}/v2/nvcf/functions" \ + -H "Host: api.${GATEWAY_ADDR}" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${NVCF_TOKEN}" \ + -d '{ + "name": "my-echo-function", + "inferenceUrl": "/echo", + "healthUri": "/health", + "inferencePort": 8000, + "containerImage": "//load_tester_supreme:0.0.8" + }' | jq . + +# Extract function and version IDs from the response +export FUNCTION_ID= +export FUNCTION_VERSION_ID= + +# Deploy the function +# Adjust instanceType and gpu based on your cluster configuration +# Instance Type Examples: NCP.GPU.A10G_1x, NCP.GPU.H100_1x, NCP.GPU.L40S_1x, etc. +# GPU Examples: A10G, H100, L40S, etc. +curl -s -X POST "http://${GATEWAY_ADDR}/v2/nvcf/deployments/functions/${FUNCTION_ID}/versions/${FUNCTION_VERSION_ID}" \ + -H "Host: api.${GATEWAY_ADDR}" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${NVCF_TOKEN}" \ + -d '{ + "deploymentSpecifications": [ + { + "instanceType": "NCP.GPU.A10G_1x", + "backend": "nvcf-default", + "gpu": "A10G", + "maxInstances": 1, + "minInstances": 1 + } + ] + }' | jq . + +# Generate an API key for invocation. See the API scope reference for endpoint-specific scope requirements. +# Set expiration to 1 day from now (required field) +EXPIRES_AT=$(date -u -v+1d '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || date -u -d '+1 day' '+%Y-%m-%dT%H:%M:%SZ') +SERVICE_ID="nvidia-cloud-functions-ncp-service-id-aketm" + +export API_KEY=$(curl -s -X POST "http://${GATEWAY_ADDR}/v1/keys" \ + -H "Host: api-keys.${GATEWAY_ADDR}" \ + -H "Content-Type: application/json" \ + -H "Key-Issuer-Service: nvcf-api" \ + -H "Key-Issuer-Id: ${SERVICE_ID}" \ + -H "Key-Owner-Id: test@nvcf-api.local" \ + -d '{ + "description": "test invocation key", + "expires_at": "'"${EXPIRES_AT}"'", + "authorizations": { + "policies": [{ + "aud": "'"${SERVICE_ID}"'", + "auds": ["'"${SERVICE_ID}"'"], + "product": "nv-cloud-functions", + "resources": [ + {"id": "*", "type": "account-functions"}, + {"id": "*", "type": "authorized-functions"} + ], + "scopes": ["invoke_function", "list_functions", "queue_details", "list_functions_details"] + }] + }, + "audience_service_ids": ["'"${SERVICE_ID}"'"] + }' | jq -r '.value') + +echo "API Key: ${API_KEY:0:20}..." + +# Wait for deployment to be ready (list functions to see status), then invoke the function +# Uses wildcard subdomain routing: .invocation. +curl -s -X POST "http://${GATEWAY_ADDR}/echo" \ + -H "Host: ${FUNCTION_ID}.invocation.${GATEWAY_ADDR}" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer ${API_KEY}" \ + -d '{"message": "hello world", "repeats": 1}' +``` + + +The `backend` value should match the cluster group name registered by the NVCA operator. +The `instanceType` and `gpu` values depend on the GPU types available in your cluster. + +For invocation, the Host header uses wildcard subdomain routing: `.invocation.`. +The URL path should match the function's `inferenceUrl` (e.g., `/echo`). +For full HTTP invocation behavior, streaming, and errors, see +[Generic HTTP Function Invocation](/nvcf/self-managed/generic-http-function-invocation). + + + +You can also use the NVCF CLI for easier function management: + +- Create, deploy, and invoke functions with simple commands +- Create or update registry credentials without manual API calls + +See [self-hosted-cli](/nvcf/self-managed/cli) for installation and usage instructions. + +## Re-registering a cluster + +Registration is performed by the CLI, not by the operator. To re-register (for +example, after a failed install, or to refresh the recorded issuer and JWKS), +re-run the compute-plane registration target and then re-apply the operator with +the refreshed values: + +```bash +make -C deploy/stacks/nvcf-compute-plane register-cluster \ + CLUSTER_NAME= \ + CLUSTER_REGION= \ + CONTROL_PLANE_PROFILE="$(pwd)/deploy/stacks/self-managed/out/control-plane-profile.yaml" \ + KUBECONFIG_FILE= \ + COMPUTE_KUBE_CONTEXT= \ + NVCF_CLI= \ + NVCF_CLI_CONFIG= + +make -C deploy/stacks/nvcf-compute-plane install \ + CLUSTER_NAME= \ + HELMFILE_ENV= \ + NCA_ID= \ + KUBECONFIG_FILE= \ + COMPUTE_KUBE_CONTEXT= +``` + +The registration target rewrites +`registration/-register-values.yaml`. The install target +copies that file into `out/` before running Helmfile. The operator reads the +cluster identity at startup, so re-running the install restarts it with the new +values. + +## Uninstalling + +To fully remove the NVCA Operator and all associated resources: + + +If functions are currently deployed on the cluster (pods in the `nvcf-backend` namespace), +undeploy them through the NVCF API or CLI before uninstalling the operator. Attempting +to delete NVCA while function pods are running can cause finalizers to block namespace +deletion. If you encounter stuck resources, see [Handling Stuck Resources] below. + + + +1. Delete the NVCFBackend resource. This triggers operator-managed cleanup of + the agent deployment, NVCA system pods, and related resources: + + ```bash + kubectl --kubeconfig \ + delete nvcfbackends --all -n nvca-operator --timeout=60s + ``` + +2. Verify the agent namespace is clean before proceeding: + + ```bash + kubectl --kubeconfig get pods -n nvca-system + + # Expected: "No resources found in nvca-system namespace." + ``` + +3. Destroy the compute-plane Helmfile release: + + ```bash + make destroy \ + CLUSTER_NAME= \ + HELMFILE_ENV= \ + NCA_ID= \ + KUBECONFIG_FILE= + ``` + + + The cluster identity (`clusterID` and `clusterGroupID`) persists in the control plane + (ICMS) and in your registration values file. A reinstall reuses it by + re-applying that file. Re-run the supported `register-cluster` target to + refresh the generated values before reinstalling. + + + +4. Delete CRDs. This removes all NVCFBackend, MiniService, and StorageRequest + custom resources cluster-wide: + + ```bash + kubectl --kubeconfig delete crd \ + nvcfbackends.nvcf.nvidia.io \ + miniservices.nvca.nvcf.nvidia.io \ + storagerequests.nvca.nvcf.nvidia.io \ + --ignore-not-found + ``` + +5. Delete namespaces: + + ```bash + kubectl --kubeconfig delete namespace \ + nvca-operator nvca-system nvcf-backend nvca-modelcache-init \ + --ignore-not-found + ``` + +### Handling Stuck Resources + +If step 1 times out and namespaces remain stuck in `Terminating` state, or function pods in +`nvcf-backend` prevent cleanup, use the [force-cleanup-script](/nvcf/self-managed/troubleshooting). This script removes +finalizers on stuck NVCA resources, force-deletes function pods, and cleans up all NVCA +namespaces. + +```bash +# Preview what will be deleted +./force-cleanup-nvcf.sh --dry-run + +# Execute the cleanup +./force-cleanup-nvcf.sh +``` + + +The force cleanup script bypasses normal cleanup procedures by removing finalizers. Always +attempt the ordered uninstall steps above first. + + + +For the full script, download link, and detailed usage instructions, see the NVCA Force +Cleanup Script appendix in the self-hosted troubleshooting guide. + +## Troubleshooting + +- Cluster IDs empty in the ConfigMap after install: The registration values were + not applied. Confirm + `registration/-register-values.yaml` exists and that its + `clusterID` and `clusterGroupID` are populated, then re-run `make install`: + + ```bash + kubectl get cm nvcfbackend-self-managed -n nvca-operator \ + -o jsonpath='{.data.cluster-dto\.yaml}' + ``` + +- Operator pod not starting: Check the operator logs: + + ```bash + kubectl logs -n nvca-operator -l app.kubernetes.io/name=nvca-operator -c nvca-operator --tail=100 + ``` + +- Operator or agent logs show `failed to create watcher` with `too many open files`: + Increase the node inotify limits with the `node-inotify-tuner` DaemonSet in + [Node inotify limits](#node-inotify-limits), then restart the affected pod. + +- NVCA agent pod not created: The operator creates the agent pod via the NVCFBackend + resource. Check the operator logs for reconciliation errors: + + ```bash + kubectl describe nvcfbackends -n nvca-operator + ``` + +- Agent fails to register with SIS (HTTP 401): The control plane could not + validate the agent's PSAT against the recorded issuer and JWKS. Re-run + `make register-cluster` for the cluster (see + [Re-registering a cluster](#re-registering-a-cluster)) so ICMS has the + current issuer and JWKS. Also verify the vault agent sidecar on the agent pod + is running and rendering the secrets file: + + ```bash + kubectl logs -n nvca-system -l app.kubernetes.io/name=nvca -c vault-agent --tail=10 + ``` + +- Vault agent sidecar failing: The agent pod needs to authenticate with OpenBao. Verify + the vault system is healthy: + + ```bash + kubectl exec -n vault-system openbao-server-0 -- bao status + ``` + +- No GPUs discovered: Ensure the GPU Operator is installed and GPU nodes have the + `nvidia.com/gpu` resource advertised: + + ```bash + kubectl get nodes -o custom-columns="NAME:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu" + ``` diff --git a/docs/compute-plane/cluster-management/topology-aware-scheduling.md b/docs/compute-plane/cluster-management/topology-aware-scheduling.md new file mode 100644 index 0000000000..c2ff8c557c --- /dev/null +++ b/docs/compute-plane/cluster-management/topology-aware-scheduling.md @@ -0,0 +1,266 @@ +# Topology-Aware Scheduling + +Topology-aware scheduling places related Pods in a hardware domain with the +network bandwidth they require. On an NVLink-optimized cluster, NVCF can place +a multi-node function inside one GPU clique instead of spreading its Pods +across slower links. + +The scheduling path has four layers: + +1. The NVIDIA GPU DRA driver labels nodes with `nvidia.com/gpu.clique`, + identifying a particular NVLink partition. +2. KAI Scheduler reads a cluster-scoped `Topology` and places a gang in the + requested domain. +3. Grove maps its workload hierarchy to the KAI topology through a `ClusterTopologyBinding`. +4. Dynamo uses Grove for placement of frontend, prefill, decode, and other + inference services. + +See the +[NVIDIA GPU DRA ComputeDomain guide](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/dra-cds.html), +[KAI topology guide](https://github.com/kai-scheduler/KAI-Scheduler/blob/main/docs/topology/README.md), +[Grove topology guide](https://github.com/NVIDIA/grove/blob/main/docs/user-guide/topology-aware-scheduling.md), +and [Dynamo topology guide](https://docs.nvidia.com/dynamo/v1.4.1/kubernetes-deployment/scale/topology-aware-scheduling) +for component-level details. + +Helm functions can also use the legacy +[`dra.nvcf.nvidia.io` partition annotation](/nvcf/self-managed/helm-functions#legacy-nvca-nvlink-partition-annotation). +That path uses Kubernetes Pod affinity and is best-effort without KAI Scheduler +or Grove topology-aware scheduling. Use the KAI or Grove mechanisms on this +page when clique placement must be coordinated for the complete workload. + +## Prerequisites + +- Install the + [NVIDIA GPU DRA driver](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/dra-intro-install.html). +- Request a full node of GPUs for each GPU-enabled Pod in your Helm chart. + The GPU DRA driver currently supports one GPU-enabled Pod per node in this mode. + +See [NVLink-optimized clusters](./configuration.md#nvlink-optimized-clusters) +for cluster registration and GPU request requirements. + +### Use KAI Scheduler only + +Enable KAI Scheduler in the compute plane stack during installation: + +```yaml +addons: + topologyAwareScheduling: + enabled: true + kaiScheduler: + enabled: true + groveOperator: + enabled: false + dynamoOperator: + enabled: false +``` + +Topology levels are ordered from the widest domain to the narrowest domain. +The compute plane automatically creates this KAI resource on installation: + +```yaml +apiVersion: kai.scheduler/v1alpha1 +kind: Topology +metadata: + name: nvcf-mnnvl-topology +spec: + levels: + - nodeLabel: nvidia.com/gpu.clique + - nodeLabel: kubernetes.io/hostname +``` + +Enabling `addons.topologyAwareScheduling` without +`addons.kaiScheduler.enabled` stops Helmfile rendering with an error. + + +The `Topology` name will always be `nvcf-mnnvl-topology` for MNNVL scheduling, +with the `nvidia.com/gpu.clique` node label for placement. +When using a KAI `PodGang` or topology annotations, ensure this name is used. + + +#### Example: place a StatefulSet in one GPU clique + +Add the KAI topology annotations to the StatefulSet object: + +```yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: + kai.scheduler/topology: "nvcf-mnnvl-topology" + kai.scheduler/topology-required-placement: "nvidia.com/gpu.clique" +spec: + podManagementPolicy: Parallel + replicas: 2 +``` + +KAI waits until all replicas fit in one value of +`nvidia.com/gpu.clique`. This is also a gang-scheduling request. See +[Gang Scheduling](./gang-scheduling.md) for atomic placement behavior and +Grove or Dynamo workloads. + +Creating a `Topology` does not change workloads that do not opt in. + +## Use Grove and/or Dynamo + +Enable the complete scheduling stack during compute stack installation +when functions use Grove or Dynamo: + +```yaml +addons: + topologyAwareScheduling: + enabled: true + kaiScheduler: + enabled: true + groveOperator: + enabled: true + dynamoOperator: + enabled: true +``` + +The `topologyAwareScheduling` toggle enables topology handling in Grove and +automatically creates a binding for each configured KAI topology on installation: + +```yaml +apiVersion: grove.io/v1alpha1 +kind: ClusterTopologyBinding +metadata: + name: nvcf-mnnvl-topology-binding +spec: + levels: + - domain: gpuclique + key: nvidia.com/gpu.clique + - domain: hostname + key: kubernetes.io/hostname + schedulerTopologyBindings: + - schedulerName: kai-scheduler + topologyReference: nvcf-mnnvl-topology +``` + +See the +[Grove automatic MNNVL guide](https://github.com/NVIDIA/grove/blob/main/docs/user-guide/auto-mnnvl.md) +for more details. + + +The `ClusterTopologyBinding` name will always be `nvcf-mnnvl-topology-binding` for MNNVL scheduling, +with the `gpuclique` domain and `nvidia.com/gpu.clique` key for placement. +When using Grove `PodClique*` or Dynamo `DynamoGraphDeployment*` family types with topology constraints, +ensure these constants are used. + + +### Example: place a Grove workload in one GPU clique + +Grove workload authors use the portable `gpuclique` domain from the generated +binding instead of the cluster's node label: + +```yaml +apiVersion: grove.io/v1alpha1 +kind: PodCliqueSet +metadata: + name: distributed-inference +spec: + replicas: 1 + template: + topologyConstraint: + topologyName: nvcf-mnnvl-topology-binding + pack: + required: gpuclique + cliques: + - name: worker + spec: + roleName: worker + replicas: 2 + minAvailable: 2 + podSpec: + containers: + - name: worker + image: + resources: + limits: + nvidia.com/gpu: "4" +``` + +Grove translates `gpuclique` to `nvidia.com/gpu.clique` and sends the +constraint to KAI. Child `PodClique` or `PodCliqueScalingGroup` constraints +can select the same or a narrower domain. See the +[Grove topology constraint rules](https://github.com/NVIDIA/grove/blob/main/docs/user-guide/topology-aware-scheduling.md#topology-constraint-rules). + +### Example: place an entire DynamoGraphDeployment in one GPU clique + +This `DynamoGraphDeployment` will create Grove pod cliques for GPU and frontend workers +that get scheduled in a single `gpuclique` domain: + +```yaml +apiVersion: nvidia.com/v1alpha1 +kind: DynamoGraphDeployment +metadata: + name: my-llm +spec: + topologyConstraint: + topologyProfile: nvcf-mnnvl-topology-binding + packDomain: gpuclique + services: + VllmWorker: + componentType: worker + replicas: 2 + envFromSecret: hf-token-secret + resources: + limits: + gpu: "1" + extraPodSpec: + mainContainer: + image: my-image + command: ["/bin/sh", "-c"] + args: + - python3 -m dynamo.vllm --model Qwen/Qwen3-0.6B + Frontend: + componentType: frontend + replicas: 1 + extraPodSpec: + mainContainer: + image: my-image + command: ["/bin/sh", "-c"] + args: + - python3 -m dynamo.frontend +``` + +In general Dynamo creates Grove resources from a `DynamoGraphDeployment`, which Grove +and KAI place. Before adding +topology constraints to a `DynamoGraphDeployment`, see the +[topology-aware scheduling guide for Dynamo 1.4.1](https://docs.nvidia.com/dynamo/v1.4.1/kubernetes-deployment/scale/topology-aware-scheduling) +to check the workload fields and topology resources expected by the compute +plane stack's pinned operator. + +## Verify topology resources (Cluster admins only) + +Confirm that nodes have GPU clique labels: + +```bash +kubectl get nodes -L nvidia.com/gpu.clique +``` + +Confirm that the KAI topology exists: + +```bash +kubectl get topologies.kai.scheduler nvcf-mnnvl-topology +``` + +When Grove is enabled, confirm that its binding exists: + +```bash +kubectl get clustertopologybindings.grove.io \ + nvcf-mnnvl-topology-binding +``` + +Grove v0.1.0-alpha.12 renamed PodGang resources from legacy base/scaled names +to an epoch-based naming scheme. The Grove operator migrates existing PodGangs +automatically on startup; running Pods are not disrupted, and no action is +required. PodGang names in `kubectl describe podgroup` output may look +different after the upgrade. + +If Pods remain `Pending`, verify that one clique has enough free nodes for the +entire gang. Then inspect the Pod and scheduler events: + +```bash +kubectl describe pod -n +kubectl describe podgroup -n +``` diff --git a/docs/user/fake-gpu-operator.md b/docs/compute-plane/fake-gpu-operator.md similarity index 100% rename from docs/user/fake-gpu-operator.md rename to docs/compute-plane/fake-gpu-operator.md diff --git a/docs/user/runbooks/caches.md b/docs/compute-plane/runbooks/caches.md similarity index 100% rename from docs/user/runbooks/caches.md rename to docs/compute-plane/runbooks/caches.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/release-notes/0.20.6.md b/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/release-notes/0.20.6.md deleted file mode 100644 index 57c41b6625..0000000000 --- a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/release-notes/0.20.6.md +++ /dev/null @@ -1,17 +0,0 @@ -# NVCF Self-Managed 0.20.6 Release Notes - -These release notes cover the following qualified stack set: - -- Control plane: `nvcf-self-managed-stack` 0.20.6 -- Compute plane: `nvcf-compute-plane-stack` 0.4.4 -- Observability: `nvcf-observability-stack` 0.2.2 - - -Detailed changes, known issues, and compatibility notes will be added before -the release is finalized. - - -## Upgrade notes - -The validated upgrade path starts from `nvcf-self-managed-stack` 0.6.1. See -[Upgrade from 0.6.1 to 0.20.6](./0.6.1-to-0.20.6-upgrade.md). diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/release-notes/index.md b/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/release-notes/index.md deleted file mode 100644 index a7698ce908..0000000000 --- a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/release-notes/index.md +++ /dev/null @@ -1,8 +0,0 @@ -# Release Notes - -Changes, fixes, and upgrade procedures for each self-managed NVCF release. - -- [0.20.6](./0.20.6.md): Release notes for the qualified control-plane 0.20.6, - compute-plane 0.4.4, and observability 0.2.2 stack set. -- [0.6.1 to 0.20.6 Upgrade](./0.6.1-to-0.20.6-upgrade.md): Upgrade procedure - from `nvcf-self-managed-stack` 0.6.1 to 0.20.6. diff --git a/docs/dev/fake-gpu-operator.md b/docs/dev/fake-gpu-operator.md index 6a89569809..4621f974d3 100644 --- a/docs/dev/fake-gpu-operator.md +++ b/docs/dev/fake-gpu-operator.md @@ -138,7 +138,7 @@ The recommended sequence is: 1. Install KWOK 2. Install fake-gpu-operator and label target nodes 3. Verify `nvidia.com/gpu` appears in node allocatable resources -4. Proceed with the [control-plane installation](../user/helmfile-installation.md) +4. Proceed with the [control-plane installation](/nvcf/self-managed/helmfile-installation) ### If Installed After the Control Plane @@ -160,7 +160,7 @@ kubectl rollout status deployment nvca-operator -n nvca-operator --timeout=120s The operator restart will re-run the bootstrap init container, recreate the NVCFBackend resource, and spawn a fresh NVCA agent pod that discovers the simulated GPUs. -For details on the bootstrap process, see [Self-Managed Clusters](../user/cluster-management/self-managed.md) (Manual Cluster +For details on the bootstrap process, see [Self-Managed Clusters](/nvcf/compute-plane/self-managed-clusters) (Manual Cluster Registration). ## Customization diff --git a/docs/dev/github-release-process.md b/docs/dev/github-release-process.md index f014a9c6f9..963eda56e2 100644 --- a/docs/dev/github-release-process.md +++ b/docs/dev/github-release-process.md @@ -270,6 +270,12 @@ what the commits since the last tag imply. NVCA declared the same pair until the 3.3 line and now uses semantic-release; see "NVCA's cutover onto semantic-release" below. +All three stacks were level-set to `1.0.0` when they moved onto this +model. Their trains advance independently from there. Nothing in the +tooling ties a compute-plane train to a self-managed or observability +train; the compatibility matrix (see "Compatibility and upgrade stops") +records which trains run together. + ### What runs where | Ref | What `auto` does | @@ -359,6 +365,41 @@ pull request's branch in the run dialog to build that pull request. branch cannot be selected and release credentials never run against unreviewed code. +### Inventory attachment is keyed by the tag + +`tag` resolves the stack that owns a pushed tag with +`release_asset_service`, which matches the tag against each service's +`tag_format` (and `legacy_tag_prefix` for the compute plane). The +matching service's `resolved_inventory_asset` is rendered at the tag and +uploaded to the draft release before it is published. The branch that +cut the tag never enters that decision, so a tag from +`release-deploy/stacks//vX.Y` attaches its inventory exactly as +a tag from `main` did before the version-file model. + +### Compatibility and upgrade stops + +- Per stack, trains N and N-1 are maintained. Older trains receive no + patches. +- Patch upgrades within a train are always supported. +- Upgrading across more than one train steps through each intermediate + train's latest patch. Every train is an upgrade stop. +- `docs/overview/compatibility-matrix.md` is the source of truth for + which trains of the three stacks run together. `tools/docs-version-sync` + generates it from the `compatibility:` block in + `docs/version-catalog/main.yaml`. A stack change that alters + cross-stack compatibility updates that block in the same change. + +### Per-stack documentation + +Each stack is a Fern product with its own version list. After a train's +first release, freeze that stack's docs alone: + +```bash +./tools/scripts/cut-docs-version.sh --stack observability --train 1.1 +``` + +The other two stacks' documentation is untouched by that cut. + ## NVCA's cutover onto semantic-release NVCA declared `version_file` and `dev_prerelease` until the 3.3 line. diff --git a/docs/dev/grpc-load-testing.md b/docs/dev/grpc-load-testing.md index 785e7a5171..2d12a8283c 100644 --- a/docs/dev/grpc-load-testing.md +++ b/docs/dev/grpc-load-testing.md @@ -5,8 +5,8 @@ ### Self-hosted CLI You need a working `nvcf-cli` configured against your self-managed cluster. -If you have not set this up yet, follow the [self-hosted-cli](../user/cli.md) guide to -install the binary and the [cli-configuration](../user/cli.md) section to point it at your +If you have not set this up yet, follow the [self-hosted-cli](/nvcf/overview/cli) guide to +install the binary and the [cli-configuration](/nvcf/overview/cli) section to point it at your gateway. Verify the CLI can reach the cluster before continuing: diff --git a/docs/dev/http-soak-testing.md b/docs/dev/http-soak-testing.md index df7e4b7272..284af486be 100644 --- a/docs/dev/http-soak-testing.md +++ b/docs/dev/http-soak-testing.md @@ -11,8 +11,8 @@ extended period (default 48 hours). ### Self-hosted CLI You need a working `nvcf-cli` configured against your self-managed cluster. -If you have not set this up yet, follow the [self-hosted-cli](../user/cli.md) guide to -install the binary and the [cli-configuration](../user/cli.md) section to point it at your +If you have not set this up yet, follow the [self-hosted-cli](/nvcf/overview/cli) guide to +install the binary and the [cli-configuration](/nvcf/overview/cli) section to point it at your gateway. Verify the CLI can reach the cluster before continuing: diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/example-dashboards.md b/docs/observability-1.0/example-dashboards.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/example-dashboards.md rename to docs/observability-1.0/example-dashboards.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/cassandra/metrics.md b/docs/observability-1.0/metrics/cassandra/metrics.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/cassandra/metrics.md rename to docs/observability-1.0/metrics/cassandra/metrics.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/ess/metrics.md b/docs/observability-1.0/metrics/ess/metrics.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/ess/metrics.md rename to docs/observability-1.0/metrics/ess/metrics.md diff --git a/docs/user/autoscaling/observability.md b/docs/observability-1.0/metrics/function-autoscaler/metrics.md similarity index 92% rename from docs/user/autoscaling/observability.md rename to docs/observability-1.0/metrics/function-autoscaler/metrics.md index 8598cbba83..cbb119a779 100644 --- a/docs/user/autoscaling/observability.md +++ b/docs/observability-1.0/metrics/function-autoscaler/metrics.md @@ -1,4 +1,4 @@ -# Function Autoscaler Observability +# Autoscaler Metrics The Function Autoscaler emits structured logs, Prometheus metrics, and OpenTelemetry spans. The chart exposes its Prometheus exporter through the @@ -67,6 +67,6 @@ Useful target prefixes: ## See also -- [Function Autoscaler Operations](./operations.md) for common symptoms tied to these metrics and log lines. -- [Architecture](./architecture.md) for the components that emit each signal. -- [Configure Autoscaling](../configure-autoscaling.md) for setting per-function scaling bounds and policy via the NVCF API. +- [Function Autoscaler Operations](/nvcf/self-managed/operations) for common symptoms tied to these metrics and log lines. +- [Architecture](/nvcf/self-managed/architecture) for the components that emit each signal. +- [Configure Autoscaling](/nvcf/overview/configure-autoscaling) for setting per-function scaling bounds and policy via the NVCF API. diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/grpc-proxy/metrics.md b/docs/observability-1.0/metrics/grpc-proxy/metrics.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/grpc-proxy/metrics.md rename to docs/observability-1.0/metrics/grpc-proxy/metrics.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/init-container/metrics.md b/docs/observability-1.0/metrics/init-container/metrics.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/init-container/metrics.md rename to docs/observability-1.0/metrics/init-container/metrics.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/invocation-service/metrics.md b/docs/observability-1.0/metrics/invocation-service/metrics.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/invocation-service/metrics.md rename to docs/observability-1.0/metrics/invocation-service/metrics.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/llm-api-gateway/metrics.md b/docs/observability-1.0/metrics/llm-api-gateway/metrics.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/llm-api-gateway/metrics.md rename to docs/observability-1.0/metrics/llm-api-gateway/metrics.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/llm-function-invocation-path.md b/docs/observability-1.0/metrics/llm-function-invocation-path.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/llm-function-invocation-path.md rename to docs/observability-1.0/metrics/llm-function-invocation-path.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/llm-request-router/metrics.md b/docs/observability-1.0/metrics/llm-request-router/metrics.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/llm-request-router/metrics.md rename to docs/observability-1.0/metrics/llm-request-router/metrics.md diff --git a/docs/user/metrics/metrics-index.md b/docs/observability-1.0/metrics/metrics-index.md similarity index 94% rename from docs/user/metrics/metrics-index.md rename to docs/observability-1.0/metrics/metrics-index.md index 66cb8056c2..98fedad442 100644 --- a/docs/user/metrics/metrics-index.md +++ b/docs/observability-1.0/metrics/metrics-index.md @@ -10,7 +10,7 @@ Per-service metrics reference for the NVCF self-hosted control plane. Each linke - [gRPC Proxy](./grpc-proxy/metrics.md): client connection counts, NATS pipe health, gRPC worker session-attach latency, and HTTP RED metrics for the gRPC proxy. - [State Metrics Service](./state-metrics/metrics.md): per-function instance count, stage durations, request latency, and function metadata. - [SIS/Spot](./sis-spot/metrics.md): HTTP client metrics for the Spot Instance Service. -- [Function Autoscaler](../autoscaling/observability.md): OpenTelemetry metrics emitted by the function autoscaler service. +- [Function Autoscaler](./function-autoscaler/metrics.md): OpenTelemetry metrics emitted by the function autoscaler service. ## LLM services diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/nvcf-api/metrics.md b/docs/observability-1.0/metrics/nvcf-api/metrics.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/nvcf-api/metrics.md rename to docs/observability-1.0/metrics/nvcf-api/metrics.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/sis-spot/metrics.md b/docs/observability-1.0/metrics/sis-spot/metrics.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/sis-spot/metrics.md rename to docs/observability-1.0/metrics/sis-spot/metrics.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/state-metrics/metrics.md b/docs/observability-1.0/metrics/state-metrics/metrics.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/state-metrics/metrics.md rename to docs/observability-1.0/metrics/state-metrics/metrics.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/utils-container/metrics.md b/docs/observability-1.0/metrics/utils-container/metrics.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/utils-container/metrics.md rename to docs/observability-1.0/metrics/utils-container/metrics.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/vault-openbao/metrics.md b/docs/observability-1.0/metrics/vault-openbao/metrics.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/vault-openbao/metrics.md rename to docs/observability-1.0/metrics/vault-openbao/metrics.md diff --git a/docs/user/observability.md b/docs/observability-1.0/observability.md similarity index 97% rename from docs/user/observability.md rename to docs/observability-1.0/observability.md index 84953d70a6..7ffaa2cdd4 100644 --- a/docs/user/observability.md +++ b/docs/observability-1.0/observability.md @@ -137,7 +137,7 @@ Profiles set defaults. Individual components can use `install`, `existing`, or The bundled VictoriaMetrics instance runs in `monitoring` by default. Configure its persistent volume in the Helmfile environment if the defaults are not -suitable. See [Helmfile Installation](./helmfile-installation.md#observability-configuration). +suitable. See [Helmfile Installation](/nvcf/self-managed/helmfile-installation#observability-configuration). Use `metricsBackend.mode: existing` to connect a customer-managed backend: @@ -158,7 +158,7 @@ for PromQL queries. Configure collector remote-write authentication separately. The shared collector discovers targets only in its Kubernetes cluster. In a split deployment, configure compute-plane collection separately. See [Cluster -Monitoring](./cluster-management/monitoring.md). +Monitoring](/nvcf/compute-plane/cluster-monitoring). ### Logging @@ -396,7 +396,7 @@ For troubleshooting common observability issues: ``` For shared stack or Function Autoscaler issues, see -[Function Autoscaler Operations](./autoscaling/operations.md). +[Function Autoscaler Operations](/nvcf/self-managed/operations). **Logs not being collected:** @@ -455,9 +455,9 @@ For shared stack or Function Autoscaler issues, see ## Related Documentation -- [Function Autoscaling](./autoscaling/index.md) -- [Function Autoscaler Observability](./autoscaling/observability.md) -- [Cluster Monitoring](./cluster-management/monitoring.md) +- [Function Autoscaling](/nvcf/self-managed/function-autoscaling-overview) +- [Function Autoscaler Observability](./metrics/function-autoscaler/metrics.md) +- [Cluster Monitoring](/nvcf/compute-plane/cluster-monitoring) - [OpenTelemetry documentation](https://opentelemetry.io/docs/) - [Prometheus documentation](https://prometheus.io/docs/) diff --git a/docs/user/example-dashboards.md b/docs/observability/example-dashboards.md similarity index 100% rename from docs/user/example-dashboards.md rename to docs/observability/example-dashboards.md diff --git a/docs/user/metrics/cassandra/metrics.md b/docs/observability/metrics/cassandra/metrics.md similarity index 100% rename from docs/user/metrics/cassandra/metrics.md rename to docs/observability/metrics/cassandra/metrics.md diff --git a/docs/user/metrics/ess/metrics.md b/docs/observability/metrics/ess/metrics.md similarity index 100% rename from docs/user/metrics/ess/metrics.md rename to docs/observability/metrics/ess/metrics.md diff --git a/docs/observability/metrics/function-autoscaler/metrics.md b/docs/observability/metrics/function-autoscaler/metrics.md new file mode 100644 index 0000000000..cbb119a779 --- /dev/null +++ b/docs/observability/metrics/function-autoscaler/metrics.md @@ -0,0 +1,72 @@ +# Autoscaler Metrics + +The Function Autoscaler emits structured logs, Prometheus metrics, and +OpenTelemetry spans. The chart exposes its Prometheus exporter through the +`function-autoscaler` service on the `metrics` port, which defaults to `41338`. +The shared stack creates a ServiceMonitor for this endpoint when +`observability.profile` is `control` or `all`. The `compute` and +`disabled` profiles do not create it. + +These service metrics describe the autoscaler itself. They are separate from +the function metrics that the autoscaler reads from VictoriaMetrics or an +external backend. + +## Metric reference + +| Metric name | Metric type | Description | +|-------------|-------------|-------------| +| `nvcf_autoscaler.autoscaling.status` | Gauge | Scaling status per function, encoded as a reason code. | +| `nvcf_autoscaler.scaling.current_instances` | Gauge | Current instance count per function as read from the timeseries database. | +| `nvcf_autoscaler.scaling.desired_instances` | Gauge | Desired instance count computed by the scaling decision. | +| `nvcf_autoscaler.scaling.utilization` | Gauge | Utilization percentage per function used in the scaling decision. | +| `nvcf_autoscaler.requests.queued_total` | Counter | Scaling requests queued for processing. | +| `nvcf_autoscaler.requests.processed_total` | Counter | Scaling requests processed. | +| `nvcf_autoscaler.requests.rejected_total` | Counter | Scaling requests rejected by the policy or guard rails. | +| `nvcf_autoscaler.requests.rate_limited_total` | Counter | Scaling requests rate-limited downstream. | +| `nvcf_autoscaler.queue.size` | Gauge | Current depth of the scaling work queue. | +| `nvcf_autoscaler.queue.capacity` | Gauge | Configured capacity of the scaling work queue. | +| `nvcf_autoscaler.function_table_state` | Gauge | State of the active function table entry per function. | +| `nvcf_autoscaler.function_discovery_duration_seconds` | Histogram | Duration of each discovery loop run. | +| `nvcf_autoscaler.timeseries_db.requests_total` | Counter | Timeseries database requests, labeled by status. | +| `nvcf_autoscaler.timeseries_db.request_duration_milliseconds` | Histogram | Timeseries database request latency. | +| `nvcf_autoscaler.timeseries_db.auth_failure_total` | Counter | Timeseries database authentication failures. | +| `nvcf_autoscaler.timeseries_db.server_side_failure_total` | Counter | Timeseries database server-side query failures. | +| `nvcf_autoscaler.nvcf_api.request_duration_milliseconds` | Histogram | NVCF API request latency. | +| `nvcf_autoscaler.oauth2_api.request_duration_milliseconds` | Histogram | OAuth2 token endpoint request latency. | +| `nvcf_autoscaler.oauth2_client.token_refresh_failure_total` | Counter | OAuth2 client token refresh failures. | +| `nvcf_autoscaler.cassandra.health_status` | Gauge | Cassandra client health. 1 indicates healthy, 0 indicates unhealthy. | +| `nvcf_autoscaler.health.overall_status` | Gauge | Overall service health status. | +| `nvcf_autoscaler.health.component_status` | Gauge | Per-component health status. | +| `nvcf_autoscaler.distributed_lock` | Gauge | State of the discovery distributed lock for this replica. | +| `nvcf_autoscaler.distributed_lock.acquisition_failures_total` | Counter | Discovery lock acquisition failures. | +| `nvcf_autoscaler.processing.utilization_data_age_milliseconds` | Histogram | Age of the utilization data used in each scaling decision. | + +## Tracing + +The function autoscaler emits OpenTelemetry spans for outbound calls to the timeseries database and the NVCF API, with the OTLP endpoint and span filter configurable under `server.tracing`. + +## Logging + +The function autoscaler writes structured logs to stdout. Set log filter directives in the `server.envfilter_directive` configuration field. The format follows the `tracing_subscriber` env filter syntax (Rust ecosystem standard): + +```yaml +server: + envfilter_directive: "server=info,rs_autoscaler=debug,rs_autoscaler::cassandra=warn,info" +``` + +Useful target prefixes: + +| Target | Covers | +|--------|--------| +| `server` | Binary entry point: startup, server lifecycle. | +| `rs_autoscaler` | Top-level function autoscaler library crate. | +| `rs_autoscaler::work` | Scaling loop, discovery loop, bucket reshuffles. | +| `rs_autoscaler::cassandra` | Cassandra client, LWT lock operations. | +| `rs_autoscaler::nvcf_api` | OAuth2, NVCF API calls. | +| `rs_autoscaler::timeseries_db` | Timeseries database query traces. | + +## See also + +- [Function Autoscaler Operations](/nvcf/self-managed/operations) for common symptoms tied to these metrics and log lines. +- [Architecture](/nvcf/self-managed/architecture) for the components that emit each signal. +- [Configure Autoscaling](/nvcf/overview/configure-autoscaling) for setting per-function scaling bounds and policy via the NVCF API. diff --git a/docs/user/metrics/grpc-proxy/metrics.md b/docs/observability/metrics/grpc-proxy/metrics.md similarity index 100% rename from docs/user/metrics/grpc-proxy/metrics.md rename to docs/observability/metrics/grpc-proxy/metrics.md diff --git a/docs/user/metrics/init-container/metrics.md b/docs/observability/metrics/init-container/metrics.md similarity index 100% rename from docs/user/metrics/init-container/metrics.md rename to docs/observability/metrics/init-container/metrics.md diff --git a/docs/user/metrics/invocation-service/metrics.md b/docs/observability/metrics/invocation-service/metrics.md similarity index 100% rename from docs/user/metrics/invocation-service/metrics.md rename to docs/observability/metrics/invocation-service/metrics.md diff --git a/docs/user/metrics/llm-api-gateway/metrics.md b/docs/observability/metrics/llm-api-gateway/metrics.md similarity index 100% rename from docs/user/metrics/llm-api-gateway/metrics.md rename to docs/observability/metrics/llm-api-gateway/metrics.md diff --git a/docs/user/metrics/llm-function-invocation-path.md b/docs/observability/metrics/llm-function-invocation-path.md similarity index 100% rename from docs/user/metrics/llm-function-invocation-path.md rename to docs/observability/metrics/llm-function-invocation-path.md diff --git a/docs/user/metrics/llm-request-router/metrics.md b/docs/observability/metrics/llm-request-router/metrics.md similarity index 100% rename from docs/user/metrics/llm-request-router/metrics.md rename to docs/observability/metrics/llm-request-router/metrics.md diff --git a/docs/observability/metrics/metrics-index.md b/docs/observability/metrics/metrics-index.md new file mode 100644 index 0000000000..98fedad442 --- /dev/null +++ b/docs/observability/metrics/metrics-index.md @@ -0,0 +1,34 @@ +# Metrics Overview + +Per-service metrics reference for the NVCF self-hosted control plane. Each linked page lists metric names, types, sources, descriptions, and the labels and filters that make the metric useful in queries and dashboards. + +## Control plane services + +- [NVCF API](./nvcf-api/metrics.md): request rates, response status codes, and log event counts for the NVCF API service. +- [Invocation Service](./invocation-service/metrics.md): HTTP request counts, durations, and invocation error metrics for the invocation path. +- [ESS](./ess/metrics.md): template rendering counters and HTTP client metrics for the Encrypted Secrets Service. +- [gRPC Proxy](./grpc-proxy/metrics.md): client connection counts, NATS pipe health, gRPC worker session-attach latency, and HTTP RED metrics for the gRPC proxy. +- [State Metrics Service](./state-metrics/metrics.md): per-function instance count, stage durations, request latency, and function metadata. +- [SIS/Spot](./sis-spot/metrics.md): HTTP client metrics for the Spot Instance Service. +- [Function Autoscaler](./function-autoscaler/metrics.md): OpenTelemetry metrics emitted by the function autoscaler service. + +## LLM services + +- [LLM API Gateway](./llm-api-gateway/metrics.md): request and routing metrics for the LLM API gateway. +- [LLM Function Invocation Metrics Report](./llm-function-invocation-path.md): end-to-end LLM invocation path report. +- [LLM Request Router](./llm-request-router/metrics.md): request router metrics for LLM traffic. + +## Per-function containers + +- [Init Container](./init-container/metrics.md): restart counts and termination reasons for function init containers. +- [Utils Container](./utils-container/metrics.md): restart counts, termination reasons, and worker service response metrics for function utils containers. + +## Datastores + +- [Cassandra](./cassandra/metrics.md): client request latency, timeouts, authentication failures, and endpoint connection metrics. +- [Vault/OpenBao](./vault-openbao/metrics.md): pointer to upstream OpenBao telemetry documentation. + +## See also + +- [Observability](../observability.md) for logging, tracing, and overall observability configuration. +- [Example Dashboards](../example-dashboards.md) for reference Grafana dashboards. diff --git a/docs/user/metrics/nvcf-api/metrics.md b/docs/observability/metrics/nvcf-api/metrics.md similarity index 100% rename from docs/user/metrics/nvcf-api/metrics.md rename to docs/observability/metrics/nvcf-api/metrics.md diff --git a/docs/user/metrics/sis-spot/metrics.md b/docs/observability/metrics/sis-spot/metrics.md similarity index 100% rename from docs/user/metrics/sis-spot/metrics.md rename to docs/observability/metrics/sis-spot/metrics.md diff --git a/docs/user/metrics/state-metrics/metrics.md b/docs/observability/metrics/state-metrics/metrics.md similarity index 100% rename from docs/user/metrics/state-metrics/metrics.md rename to docs/observability/metrics/state-metrics/metrics.md diff --git a/docs/user/metrics/utils-container/metrics.md b/docs/observability/metrics/utils-container/metrics.md similarity index 100% rename from docs/user/metrics/utils-container/metrics.md rename to docs/observability/metrics/utils-container/metrics.md diff --git a/docs/user/metrics/vault-openbao/metrics.md b/docs/observability/metrics/vault-openbao/metrics.md similarity index 100% rename from docs/user/metrics/vault-openbao/metrics.md rename to docs/observability/metrics/vault-openbao/metrics.md diff --git a/docs/observability/observability.md b/docs/observability/observability.md new file mode 100644 index 0000000000..7ffaa2cdd4 --- /dev/null +++ b/docs/observability/observability.md @@ -0,0 +1,473 @@ +# Observability Configuration + +This page provides guidance on configuring observability for self-hosted NVCF control-plane, including metrics, logging, and tracing. + +## Find the answer to your question + +Common operator questions and where to look on this page or in linked references. + +| Question | Where to look | +|----------|---------------| +| How do I see application-level NVCF stats (number of functions, queue depth, request latency)? | [State Metrics Service metrics](./metrics/state-metrics/metrics.md). The page documents per-function instance count, queue depth, and request latency, plus other function-level signals. | +| How do I debug a single request end-to-end? | Combine the per-hop signals: enable tracing per [Tracing Configuration](#tracing-configuration), correlate with the [Metrics Overview](./metrics/metrics-index.md) for each service in the request path, and tail the matching service logs. A consolidated hop-by-hop walkthrough is in development. | +| Where are per-service metrics? | [Metrics Overview](./metrics/metrics-index.md). | +| Where are gRPC proxy metrics? | [gRPC Proxy metrics](./metrics/grpc-proxy/metrics.md). The page documents client connection counts, NATS pipe health, gRPC worker session-attach latency, and HTTP RED metrics. | +| How do I add custom spans or metrics in a Kit application? | Use the OpenTelemetry API directly, the OmniTrace helper, the Carbonite static metrics API, or the `omni::observability::IMeter` interface. Refer to the Omniverse Kit and Carbonite documentation for details. | +| Where are reference dashboards? | [Example dashboards](./example-dashboards.md) and the [Dashboards](#dashboards) section below. | +| How do I configure the shared metrics stack? | See [Self-managed metrics stack](#self-managed-metrics-stack). | + +## Overview + +Self-hosted NVCF control-plane observability enables users to monitor the health and performance of their NVCF deployment. The observability solution is designed to be: + +- **Cloud-agnostic**: Works in any Kubernetes environment (cloud provider, on-premises, or air-gapped) +- **Offline-capable**: Fully functional in isolated networks without external dependencies +- **Bring-Your-Own (BYO)**: Integrates with your existing observability platforms +- **No vendor lock-in**: Uses open standards (Prometheus, OpenTelemetry, OTLP) + +The observability solution currently provides: + +- [Metrics Collection]: Prometheus-compatible metrics from all control-plane services +- [Logging]: Logs emitted to stdout/stderr for easy collection +- [Tracing]: Distributed tracing via OTLP to your collector +- [Dashboards]: Reference Grafana dashboards for key metrics + + +**Looking for a quick start?** If you want to quickly deploy example observability components +to explore metrics, logs, and dashboards, see [self-hosted-example-dashboards](./example-dashboards.md). + +The example deployments are designed for development and testing only, and are not suitable +for production use. For production deployments, use the self-managed metrics stack or +integrate with your own observability infrastructure. + + + +## Early Access Phase + +NVCF self-hosted observability is currently in Early Access (EA). During EA, NVCF provides interfaces and documentation for you to integrate with your own observability backend: + +**What's Provided:** + +- Documented metrics for critical control-plane services +- Example scrape targets for prometheus-operator ServiceMonitor configuration +- Metrics exposed via Prometheus-compatible endpoints +- Shared metrics collection with bundled VictoriaMetrics or an existing backend +- Logs emitted to stdout/stderr for easy collection +- Configuration and deployment documentation +- Example dashboards for key metrics + +**Your Responsibility:** + +- Configure storage for bundled VictoriaMetrics or connect an existing backend +- Configure compute-plane collection for split deployments +- Deploy log collectors (e.g., Fluentd, Promtail, OTel Collector) to aggregate logs +- Set up your preferred visualization and alerting tools + +## Control-Plane Services + +The following control-plane services expose metrics and logs for monitoring: + +**Core NVCF Services:** + +- **NVCF API**: Main API for function management and invocation +- **Invocation Service**: Handles function invocation requests +- **SPOT Instance Service (SIS)**: Manages worker pod and cluster state +- **State Metrics Service**: Aggregates and exports NVCF-specific metrics +- **Function Autoscaler**: Calculates desired function instance counts + +**Supporting Services:** + +- **Cassandra (C\*)**: Primary database for control-plane state +- **OpenBao/Vault**: Secret management and S2S authentication +- **Encrypted Secrets Service (ESS)**: Function and account secrets +- **NATS Core**: Pub/sub messaging +- **NATS JetStream**: Persistent messaging + +**Worker Pod Components:** + +- **Utils Container**: Proxy to NATS from user applications +- **Init Container**: Setup and resource loading +- **Inference Container**: Inference workload + +## Architecture + +### Metrics Collection + +All control-plane services expose Prometheus-compatible metrics endpoints. You can scrape these metrics using: + +- **Prometheus Operator**: Create ServiceMonitor resources based on the provided scrape targets +- **Prometheus**: Configure scrape targets manually +- **OpenTelemetry Collector**: Use the Prometheus receiver + +**Metrics Documentation:** + +Detailed metrics documentation is available for each service, including metric names, +types, labels, and descriptions. See the per-service metrics reference under the +`Metrics` section. + +### Self-managed metrics stack + +The Helmfile stack uses an observability profile to select the default metrics +components and monitor targets: + +| Profile | Shared monitor defaults | Function Autoscaler | NVCA observability defaults | +| --- | --- | --- | --- | +| `disabled` | None | Not installed | Disabled | +| `control` | Control-plane services | Installed | Disabled | +| `compute` | NVCA, DCGM, and worker pods | Not installed | Enabled | +| `all` | Control-plane and compute-plane targets | Installed | Enabled | + +The control-plane stack defaults to `control`. The compute-plane stack defaults +to `compute`. Use `all` when both sets of targets run in the same cluster. + +The default `control` profile installs the Prometheus Operator custom resource +definitions, OpenTelemetry Operator, collector with Target Allocator and +discovery RBAC, default control-plane monitors, and VictoriaMetrics. It also +installs State Metrics, then the Function Autoscaler. + +Profiles set defaults. Individual components can use `install`, `existing`, or +`disabled` where supported. The available layouts are: + +| Layout | Configuration | +| --- | --- | +| Bundled backend | Use the default `install` modes and review the VictoriaMetrics persistent volume settings. | +| External backend | Set `metricsBackend.mode: existing` and provide remote-write and PromQL endpoints. | +| Existing components | Set the applicable `observability.components.*.mode` values to `existing`. | +| No shared stack | Use the `disabled` profile. The Function Autoscaler is not installed. | + +The bundled VictoriaMetrics instance runs in `monitoring` by default. Configure +its persistent volume in the Helmfile environment if the defaults are not +suitable. See [Helmfile Installation](/nvcf/self-managed/helmfile-installation#observability-configuration). + +Use `metricsBackend.mode: existing` to connect a customer-managed backend: + +```yaml +metricsBackend: + mode: existing + type: external + remoteWriteEndpoint: https://metrics.example.com/write + promqlEndpoint: https://metrics.example.com + authentication: + mode: none +``` + +The collector requires the remote-write endpoint. The `control` and `all` +profiles also require the PromQL endpoint because the Function Autoscaler +queries it. The autoscaler supports `none`, `token`, and `mtls` authentication +for PromQL queries. Configure collector remote-write authentication separately. + +The shared collector discovers targets only in its Kubernetes cluster. In a +split deployment, configure compute-plane collection separately. See [Cluster +Monitoring](/nvcf/compute-plane/cluster-monitoring). + +### Logging + +**Log Format:** + +- All services emit logs to stdout/stderr (standard for Kubernetes) +- Sensitive data redaction must be configured by the log collector + +**Log Collection:** + +You can collect logs using any Kubernetes-compatible log aggregator: + +- Fluentd or Fluent Bit +- Promtail (for Loki) +- Filebeat (for Elasticsearch) +- OpenTelemetry Collector (filelog receiver) + +**System Logs:** + +System logs are available at standard UNIX locations and from the systemd journal. + +### Tracing (Available in GA) + +Distributed tracing support via OpenTelemetry Protocol (OTLP) is planned for a future release: + +- Key flows will be instrumented with OpenTelemetry SDK +- Traces will be exportable via OTLP (HTTP or gRPC) +- Configurable sampling strategies +- Support for any OTLP-compatible backend (Jaeger, Tempo, Zipkin, etc.) +- Tracing is configurable via Helm values under `global.observability.tracing` + +## Configuration + +You can use the shared metrics stack or integrate with your own backend. + +### Metrics Scraping + +The observability profile configures the shared collector and default monitors. +Some service charts also use `global.observability.metrics.enabled` to enable +their own metrics exports or PodMonitors. Set it separately when those service +metrics are needed. + +Use Prometheus Operator with the provided ServiceMonitor examples: + +```yaml +# Example ServiceMonitor for NVCF API +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: nvcf-api + namespace: nvcf +spec: + selector: + matchLabels: + app: nvcf-api + endpoints: + # Endpoint created based on the scrape target in the + # per-service metrics documentation + - port: metrics + interval: 30s + path: /metrics +``` + +Or configure Prometheus scrape targets manually in your prometheus.yml. + +#### Application-level NVCF stats + +The State Metrics Service exposes per-function signals you can query in +Prometheus. The following PromQL examples cover the three most common +operator questions. Metric names and labels are sourced from +[State Metrics Service metrics](./metrics/state-metrics/metrics.md). + +Number of registered functions: + +```promql +# nvcf_function_info is emitted per function with descriptive labels. +# Dedupe by function_id so multiple label series do not inflate the count. +count(count by (function_id) (nvcf_function_info)) +``` + +Queue depth per function: + +```promql +# nvcf_function_queue_depth is a gauge keyed by function_id. +sum by (function_id, name) (nvcf_function_queue_depth) +``` + +Function request latency (p50 and p95) over a 5-minute window: + +```promql +# p50 +histogram_quantile( + 0.50, + sum by (le, function_id) (rate(function_request_latency_bucket[5m])) +) + +# p95 +histogram_quantile( + 0.95, + sum by (le, function_id) (rate(function_request_latency_bucket[5m])) +) +``` + +### Log Collection + +Deploy a log collector as a DaemonSet to ship logs to your backend: + +```bash +# Example: Deploy Promtail for Loki +kubectl apply -f promtail-daemonset.yaml + +# Example: Deploy Fluentd or Fluent Bit +kubectl apply -f fluentd-daemonset.yaml +``` + +Configure your log collector to: + +- Tail logs from all namespaces +- Add metadata labels (pod name, namespace, service) +- Forward to your log aggregation backend (Loki, Elasticsearch, etc.) + +### Tracing Configuration + +Enable distributed tracing by setting Helm values under +`global.observability.tracing`. The control-plane exports traces via OTLP +to your own OTLP-compatible collector. Set `collectorEndpoint`, +`collectorPort`, and `collectorProtocol` to match your collector's address. +`collectorProtocol` is the endpoint URI scheme expected by the stack, not the +OTLP transport. + +Helm overrides example: + +```yaml +global: + observability: + tracing: + enabled: true + collectorEndpoint: "otel-collector-gateway-collector.observability.svc.cluster.local" + collectorPort: 4317 + collectorProtocol: http +``` + +Configuration fields: + +- `enabled`: Set to `true` to enable OTLP trace export from control-plane + services. +- `collectorEndpoint`: DNS name or address of your OTLP collector (e.g., + OpenTelemetry Collector, Jaeger collector). Use a Kubernetes service DNS name + such as `..svc.cluster.local` when the collector runs + in-cluster. +- `collectorPort`: Port on which the collector accepts OTLP traffic (e.g., + 4317 for gRPC, 4318 for HTTP depending on your collector setup). +- `collectorProtocol`: URI scheme used to build the collector endpoint + (`http` or `https`). This value does not select the OTLP transport. + +Ensure your collector is deployed and reachable from the NVCF control-plane +namespace, and that it forwards traces to your backend (Jaeger, Tempo, Zipkin, +or another OTLP-compatible system). + +## Dashboards + +Reference Grafana dashboards are provided for control-plane services showing critical metrics for key services: + +- ESS (Encrypted Secrets Service) + +- Cassandra + +- Vault + +- Invocation Service + +- NVCF API + +- SIS (SPOT Instance Service) + +- Worker Pods (Utils Container, Init Container, Inference Container) + + - Note: Worker Pods are deployed in the backend cluster, not the control-plane + cluster, but their configuration is globally controlled as part of the control-plane + +- State Metrics Service + +**Dashboard Location:** + +Dashboards are provided in native Grafana JSON format for [file-provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards). + +Load dashboards into Grafana by placing them in `/etc/grafana/provisioning/dashboards/` on startup. + +Published dashboards will be available in the +[NVCF examples](https://github.com/NVIDIA/nvcf/tree/main/examples) public GitHub repository. + +## Troubleshooting + +For troubleshooting common observability issues: + +**Metrics not appearing:** + +1. Verify the service is exposing metrics: + + ```bash + # Port-forward to the service metrics port + kubectl port-forward -n nvcf svc/nvcf-api 8080:8080 + + # In another terminal, curl the metrics endpoint + curl http://localhost:8080/metrics + ``` + +2. Check ServiceMonitor or scrape configuration: + + ```bash + # Verify ServiceMonitor exists + kubectl get servicemonitor -n nvcf + + # Check ServiceMonitor details + kubectl describe servicemonitor nvcf-api -n nvcf + ``` + +3. Verify network policies allow scraping: + + ```bash + # List network policies that might block traffic + kubectl get networkpolicy -n nvcf + + # Test connectivity from Prometheus namespace + kubectl run -n --rm -it debug \ + --image=curlimages/curl --restart=Never -- \ + curl http://nvcf-api.nvcf.svc.cluster.local:8080/metrics + ``` + +4. Check service logs for errors: + + ```bash + # Check for metrics-related errors + kubectl logs -n nvcf deployment/nvcf-api | grep -i metric + ``` + +For shared stack or Function Autoscaler issues, see +[Function Autoscaler Operations](/nvcf/self-managed/operations). + +**Logs not being collected:** + +1. Verify log collector DaemonSet is running: + + ```bash + # Check DaemonSet status (e.g., for Fluentd/Fluent Bit) + # Note: Namespaces may be different depending on the log collector deployment + kubectl get daemonset -n logging + kubectl get pods -n logging -l app=fluent-bit + ``` + +2. Check collector can access pod logs: + + ```bash + # Verify log collector has proper volume mounts + kubectl describe daemonset fluent-bit -n logging | grep -A5 Mounts + + # Check collector logs for errors + kubectl logs -n logging -l app=fluent-bit --tail=50 + ``` + +3. Verify log backend is reachable: + + ```bash + # Test connectivity to log backend (e.g. Loki) + kubectl run -n logging --rm -it debug \ + --image=curlimages/curl --restart=Never -- \ + curl -v http://loki.logging.svc.cluster.local:3100/ready + ``` + +4. Check for log redaction or filtering rules: + + ```bash + # Review collector configuration + kubectl get configmap fluent-bit-config -n logging -o yaml + + # Check if logs are being dropped + kubectl logs -n logging -l app=fluent-bit | grep -i "drop\|filter" + ``` + +## Security + +**Metrics Endpoints:** + +- Metrics endpoints should be accessed over HTTP in-cluster only + + - Any external access should be SSL/TLS or mTLS secured with a reverse proxy or other ingress controller, or + - Aggregated locally and exposed via a secured otel-collector + +- All sensitive log data should be redacted by the log collector (currently, this is the responsibility of the log collector, not the service) + + - Example implementation by OTEL Collector: [Log Redaction](https://opentelemetry.io/docs/languages/dotnet/logs/redaction/) + +- User-provided observability backend should be properly secured with RBAC, TLS/SSL, and other security best practices. + +## Related Documentation + +- [Function Autoscaling](/nvcf/self-managed/function-autoscaling-overview) +- [Function Autoscaler Observability](./metrics/function-autoscaler/metrics.md) +- [Cluster Monitoring](/nvcf/compute-plane/cluster-monitoring) +- [OpenTelemetry documentation](https://opentelemetry.io/docs/) +- [Prometheus documentation](https://prometheus.io/docs/) + +## Version Compatibility + +NVCF self-hosted control-plane observability is compatible with: + +- Supported versions are the latest Kubernetes minor release and the two prior minor releases (N-2). See official Kubernetes docs for current supported [versions](https://kubernetes.io/releases/version-skew-policy/#supported-versions). +- Any Prometheus-compatible metrics collection system +- Any log aggregation system that can collect from Kubernetes stdout/stderr or read + from the filesystem (depending on K8s cluster configuration) + +For the latest compatibility information, see the release notes. diff --git a/docs/user/api.md b/docs/overview/api.md similarity index 97% rename from docs/user/api.md rename to docs/overview/api.md index b45a8a4648..33bed46232 100644 --- a/docs/user/api.md +++ b/docs/overview/api.md @@ -1,6 +1,6 @@ # API -This page provides a brief overview of the NVCF API. All API endpoints are served through your gateway. See [gateway-routing](./gateway-routing.md) for details on configuring your gateway domain and DNS. +This page provides a brief overview of the NVCF API. All API endpoints are served through your gateway. See [gateway-routing](/nvcf/self-managed/gateway-routing) for details on configuring your gateway domain and DNS. ## OpenAPI Specification @@ -9,7 +9,7 @@ This page does not cover all endpoints. Please refer to the [OpenAPI Spec](https://api.nvcf.nvidia.com/v3/openapi) for the latest API information. -The OpenAPI spec linked above documents the full NVCF API surface. Replace the hosted domain with your own gateway domain when making requests. See [gateway-routing](./gateway-routing.md) for your deployment's base URL. +The OpenAPI spec linked above documents the full NVCF API surface. Replace the hosted domain with your own gateway domain when making requests. See [gateway-routing](/nvcf/self-managed/gateway-routing) for your deployment's base URL. diff --git a/docs/user/cli.md b/docs/overview/cli.md similarity index 97% rename from docs/user/cli.md rename to docs/overview/cli.md index 9ece976581..e1f19fc4a1 100644 --- a/docs/user/cli.md +++ b/docs/overview/cli.md @@ -71,7 +71,7 @@ selection explicit. ### Download from NGC The CLI is available as a resource from NGC. See -[download-nvcf-cli](./image-mirroring.md) for detailed download and extraction +[download-nvcf-cli](/nvcf/overview/image-mirroring) for detailed download and extraction instructions. The downloaded package includes: @@ -110,7 +110,7 @@ For self-hosted deployments, the CLI must be configured to communicate with your gateway. The gateway uses hostname-based routing for HTTP services. -For Gateway routing details, including architecture diagrams, verification commands, and production DNS/HTTPS setup, see [gateway-routing](./gateway-routing.md). +For Gateway routing details, including architecture diagrams, verification commands, and production DNS/HTTPS setup, see [gateway-routing](/nvcf/self-managed/gateway-routing). @@ -121,7 +121,7 @@ the CLI. The CLI calls the configured API, API Keys, invocation, and gRPC endpoints during token minting, cluster registration, health checks, and function operations. -Complete [Gateway quickstart](./gateway-routing.md#gateway-quickstart) before you +Complete [Gateway quickstart](/nvcf/self-managed/gateway-routing#gateway-quickstart) before you configure the CLI. That procedure installs the Gateway API CRDs, creates and labels the required namespaces, installs Envoy Gateway, creates the GatewayClass and Gateway, waits for the Gateway to be programmed, and exports: @@ -305,7 +305,7 @@ api_keys_service_url: "https://api-keys.nvcf.example.com" ``` -For complete instructions on setting up DNS records and TLS certificates, see [production-dns-https](./gateway-routing.md) in the Gateway Routing guide. +For complete instructions on setting up DNS records and TLS certificates, see [production-dns-https](/nvcf/self-managed/gateway-routing) in the Gateway Routing guide. @@ -359,7 +359,7 @@ Or use the `--debug` flag or `NVCF_DEBUG=true` environment variable per-command. ``` -For immediate testing, you can use `load_tester_supreme` from `nvcf-onprem` (see [self-hosted-artifact-manifest](./manifest.md)), which supports the `{"message": "hello world"}` request body above. For more function samples, see the [NVCF examples](https://github.com/NVIDIA/nvcf/tree/main/examples) repository and [function-creation](./function-creation.md) for function creation documentation. +For immediate testing, you can use `load_tester_supreme` from `nvcf-onprem` (see [self-hosted-artifact-manifest](/nvcf/overview/manifest)), which supports the `{"message": "hello world"}` request body above. For more function samples, see the [NVCF examples](https://github.com/NVIDIA/nvcf/tree/main/examples) repository and [function-creation](./function-creation.md) for function creation documentation. @@ -464,7 +464,7 @@ Default task key scopes: ### Self-hosted Deployment Commands -Use these commands to install and inspect self-hosted NVCF deployments. For the local k3d installation flow, see [Quickstart](./quickstart.md). +Use these commands to install and inspect self-hosted NVCF deployments. For the local k3d installation flow, see [Quickstart](/nvcf/overview/quickstart). | Command | Description | | --- | --- | @@ -503,7 +503,7 @@ Bundle sources: `self-hosted up` supports only a single local k3d cluster. It requires `--env local`, a current `k3d-*` kube context, and no split-context flags. For separate control-plane and GPU clusters, use the explicit control-plane and -compute-plane install primitives with [Self-Managed Clusters](./cluster-management/self-managed.md). +compute-plane install primitives with [Self-Managed Clusters](/nvcf/compute-plane/self-managed-clusters). ### Cluster Registration @@ -579,7 +579,7 @@ the cluster; the NVCA Operator chart does not consume it. For load-balancer-fronted gateways that route by hostname, add the matching host-header overrides (`selfManaged.icmsServiceHostHeaderOverride`, `selfManaged.revalServiceHostHeaderOverride`, `selfManaged.natsHostOverride`) to these -values. See [self-managed-clusters](./cluster-management/self-managed.md) for how the +values. See [self-managed-clusters](/nvcf/compute-plane/self-managed-clusters) for how the register values feed the operator install and when host-header overrides are required. List the self-hosted cluster registrations stored in ICMS with the admin token: @@ -1054,7 +1054,7 @@ Additional `function invoke` flags: ### Registry Credentials Commands -Manage container registry credentials for function images and Helm charts. For comprehensive setup instructions including IAM configuration for AWS ECR, see [third-party-registries-self-hosted](./third-party-registries.md). +Manage container registry credentials for function images and Helm charts. For comprehensive setup instructions including IAM configuration for AWS ECR, see [third-party-registries-self-hosted](/nvcf/self-managed/third-party-registries). | Command | Description | | --- | --- | @@ -1102,7 +1102,7 @@ Manage container registry credentials for function images and Helm charts. For c ``` -Registry credential changes take up to about 5 minutes to take effect for task creation. `nvcf-cli registry-credential list` and `get` show the new value immediately, but task processing caches account credentials for about 5 minutes (`nvct.nvcf.cache-ttl`), so a task can keep using the previous value until the cache refreshes. After rotating or deleting a credential, allow up to about 5 minutes, or restart the task service to apply it immediately. See [Credential Propagation Delay](./third-party-registries.md). +Registry credential changes take up to about 5 minutes to take effect for task creation. `nvcf-cli registry-credential list` and `get` show the new value immediately, but task processing caches account credentials for about 5 minutes (`nvct.nvcf.cache-ttl`), so a task can keep using the previous value until the cache refreshes. After rotating or deleting a credential, allow up to about 5 minutes, or restart the task service to apply it immediately. See [Credential Propagation Delay](/nvcf/self-managed/third-party-registries). ## Troubleshooting @@ -1179,4 +1179,4 @@ Registry credential changes take up to about 5 minutes to take effect for task c | `task results` | `NVCF_NVCT_API_KEY` | `list_results` | `NVCF_NVCT_API_KEY` | | `task update-secrets` | `NVCF_NVCT_API_KEY` | `update_secrets` | `NVCF_NVCT_API_KEY` | -For additional troubleshooting, see [self-hosted-troubleshooting](./troubleshooting.md). +For additional troubleshooting, see [self-hosted-troubleshooting](/nvcf/self-managed/troubleshooting). diff --git a/docs/overview/compatibility-matrix.md b/docs/overview/compatibility-matrix.md new file mode 100644 index 0000000000..5be1563675 --- /dev/null +++ b/docs/overview/compatibility-matrix.md @@ -0,0 +1,36 @@ +# Compatibility Matrix + +NVCF ships as three independently versioned Helm stacks: the self-managed +control plane, the compute plane, and the observability stack. Each stack +releases from its own release train and publishes its own documentation +version. Use this page to pick stack versions that are qualified to run +together. + +Releases are listed as trains (`X.Y`); any patch release on a train counts. +A requirement such as `1.0 or later` means that train and every later +maintained train. Only the latest train and the one before it are maintained; +upgrade to a maintained train before moving further. + +{/*docs-version-sync:BEGIN compatibility-matrix*/} + +## Current stack releases + +| Stack | Latest release | Source tag | +| --- | --- | --- | +| [Self-managed (control plane)](/nvcf/self-managed/) | `1.0.0` | `deploy/stacks/self-managed/v1.0.0` | +| [Compute plane](/nvcf/compute-plane/) | `1.0.0` | `deploy/stacks/nvcf-compute-plane/v1.0.0` | +| [Observability](/nvcf/observability/) | `1.0.0` | `deploy/stacks/observability/v1.0.0` | + +## Compatible stack versions + +| Stack | Release | Works with | +| --- | --- | --- | +| Self-managed (control plane) | `1.0` | Compute plane `1.0` or later, Observability `1.0` or later | +| Compute plane | `1.0` | Self-managed (control plane) `1.0` or later, Observability `1.0` or later | +| Observability | `1.0` | Self-managed (control plane) `1.0` or later, Compute plane `1.0` or later | + +{/*docs-version-sync:END compatibility-matrix*/} + +Stack versions above are read from the latest published GitHub release of +each stack. Documentation for each stack version is available from the +version menu on that stack's tab. diff --git a/docs/user/configure-autoscaling.md b/docs/overview/configure-autoscaling.md similarity index 95% rename from docs/user/configure-autoscaling.md rename to docs/overview/configure-autoscaling.md index 816afb4fde..31ca5eb2c0 100644 --- a/docs/user/configure-autoscaling.md +++ b/docs/overview/configure-autoscaling.md @@ -1,6 +1,6 @@ # Configure Function Autoscaling -This page explains how to configure autoscaling on a deployed function using the NVCF API. For background on how the function autoscaler decides on instance counts, see the [Function Autoscaling Overview](./autoscaling/index.md). For the full schema of the request and response bodies referenced below, see the [NVCF OpenAPI specification](https://api.nvcf.nvidia.com/v3/openapi). +This page explains how to configure autoscaling on a deployed function using the NVCF API. For background on how the function autoscaler decides on instance counts, see the [Function Autoscaling Overview](/nvcf/self-managed/function-autoscaling-overview). For the full schema of the request and response bodies referenced below, see the [NVCF OpenAPI specification](https://api.nvcf.nvidia.com/v3/openapi). Use these two endpoints: @@ -164,6 +164,6 @@ The function version uses the platform's autoscaling policy after the policy cac ## See Also -- [Function Autoscaling Overview](./autoscaling/index.md) for what the function autoscaler does and what it depends on. +- [Function Autoscaling Overview](/nvcf/self-managed/function-autoscaling-overview) for what the function autoscaler does and what it depends on. - [CLI](./cli.md) for `nvcf-cli function deploy create` and `nvcf-cli function deploy update`, which wrap the same API surface. - [NVCF OpenAPI specification](https://api.nvcf.nvidia.com/v3/openapi) for the full request and response schema of the endpoints used on this page. diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/container-functions.md b/docs/overview/container-functions.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/container-functions.md rename to docs/overview/container-functions.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/container-tasks.md b/docs/overview/container-tasks.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/container-tasks.md rename to docs/overview/container-tasks.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/function-creation.md b/docs/overview/function-creation.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/function-creation.md rename to docs/overview/function-creation.md diff --git a/docs/user/generic-http-function-invocation.md b/docs/overview/generic-http-function-invocation.md similarity index 98% rename from docs/user/generic-http-function-invocation.md rename to docs/overview/generic-http-function-invocation.md index 687a8a18e8..91e2d63d84 100644 --- a/docs/user/generic-http-function-invocation.md +++ b/docs/overview/generic-http-function-invocation.md @@ -250,7 +250,7 @@ can still use the platform error response format. For platform API behavior, see Emit logs from your inference container so invocation failures can be diagnosed. -See [Observability](./observability.md) and [Troubleshooting](./troubleshooting.md) +See [Observability](/nvcf/observability/observability) and [Troubleshooting](/nvcf/self-managed/troubleshooting) for logging and debugging guidance. diff --git a/docs/user/grpc-function-invocation.md b/docs/overview/grpc-function-invocation.md similarity index 95% rename from docs/user/grpc-function-invocation.md rename to docs/overview/grpc-function-invocation.md index 292126a4a7..557c14159c 100644 --- a/docs/user/grpc-function-invocation.md +++ b/docs/overview/grpc-function-invocation.md @@ -5,12 +5,12 @@ gRPC service. gRPC functions use the gRPC proxy instead of the HTTP invocation route. In self-hosted deployments, the gRPC route is exposed on the Gateway TCP -listener. See [Gateway Routing](./gateway-routing.md) for listener and DNS +listener. See [Gateway Routing](/nvcf/self-managed/gateway-routing) for listener and DNS configuration. Self-hosted split or multi-cluster deployments require additional enablement before workers can reach the grpc-proxy callback endpoint. See -[gRPC Invocation Enablement](./grpc-invocation-enablement.md). +[gRPC Invocation Enablement](/nvcf/self-managed/g-rpc-invocation-enablement). ## Invocation Path @@ -160,5 +160,5 @@ gRPC responses. This error does not indicate a control-plane problem. The client should discard the stale request ID and reconnect without it to start a new session. -See [Troubleshooting](./troubleshooting.md#grpc-session-resumption-fails) +See [Troubleshooting](/nvcf/self-managed/troubleshooting#grpc-session-resumption-fails) for diagnosis steps. diff --git a/docs/user/grpc-load-test-sli-guide.md b/docs/overview/grpc-load-test-sli-guide.md similarity index 100% rename from docs/user/grpc-load-test-sli-guide.md rename to docs/overview/grpc-load-test-sli-guide.md diff --git a/docs/user/grpc-load-testing.md b/docs/overview/grpc-load-testing.md similarity index 100% rename from docs/user/grpc-load-testing.md rename to docs/overview/grpc-load-testing.md diff --git a/docs/user/helm-functions.md b/docs/overview/helm-functions.md similarity index 96% rename from docs/user/helm-functions.md rename to docs/overview/helm-functions.md index e751a835cb..d3db55cc55 100644 --- a/docs/user/helm-functions.md +++ b/docs/overview/helm-functions.md @@ -81,13 +81,13 @@ A sample helm chart for a multi-node deployment can be found [in the multi-node The compute cluster must have the `NVLinkOptimized` attribute. See -[NVLink-optimized clusters](./cluster-management/configuration.md#nvlink-optimized-clusters). +[NVLink-optimized clusters](/nvcf/compute-plane/cluster-configuration#nvlink-optimized-clusters). NVCF can place the Pods from one multi-node Helm function in a single NVLink -GPU clique. Use [Gang Scheduling](./cluster-management/gang-scheduling.md) when every Pod +GPU clique. Use [Gang Scheduling](/nvcf/compute-plane/gang-scheduling) when every Pod must be placed atomically. Use -[Topology-Aware Scheduling](./cluster-management/topology-aware-scheduling.md) +[Topology-Aware Scheduling](/nvcf/compute-plane/topology-aware-scheduling) to place that gang in one GPU clique. The guides include examples for direct KAI StatefulSets and workloads managed through Grove and Dynamo. NVCA will create a @@ -109,8 +109,8 @@ NVLink partition placement through these legacy affinity rules is best-effort without KAI Scheduler or Grove topology-aware scheduling. The rules do not provide atomic gang placement. Concurrent Pods can initially land in different cliques, and distinct logical groups can land in the same clique. Use -[Gang Scheduling](./cluster-management/gang-scheduling.md) with -[Topology-Aware Scheduling](./cluster-management/topology-aware-scheduling.md) +[Gang Scheduling](/nvcf/compute-plane/gang-scheduling) with +[Topology-Aware Scheduling](/nvcf/compute-plane/topology-aware-scheduling) when the workload requires all Pods to fit and start in a specific topology domain. @@ -301,7 +301,7 @@ The following timeouts govern when a still-unhealthy object fails the instance. The clock for each timeout starts when the described condition is first observed, measured from the Pod launch time unless noted. Values are NVCA defaults. Only Worker Degradation Period is operator-configurable, through the -`Worker Degradation Period` setting in [NVCA Configuration](./cluster-management/configuration.md). +`Worker Degradation Period` setting in [NVCA Configuration](/nvcf/compute-plane/cluster-configuration). | Timeout | Default | Cause (when the clock starts) | Effect (when exceeded) | | --- | --- | --- | --- | @@ -389,7 +389,7 @@ indicating readiness. With this flag enabled: debugging, and Kubernetes can replace it. - If the health endpoint begins to not report ready after the instance has entered the `RUNNING` state, the instance is marked degraded until it reports ready again or 30 minutes have passed - (Worker Degradation Period, see [NVCA Configuration](./cluster-management/configuration.md)), + (Worker Degradation Period, see [NVCA Configuration](/nvcf/compute-plane/cluster-configuration)), after which the instance is killed and re-created by NVCF. (Note: initial startup behavior with a 2 hour timeout remains the same as default instance health behavior) diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/helm-tasks.md b/docs/overview/helm-tasks.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/helm-tasks.md rename to docs/overview/helm-tasks.md diff --git a/docs/user/http-load-test-sli-guide.md b/docs/overview/http-load-test-sli-guide.md similarity index 100% rename from docs/user/http-load-test-sli-guide.md rename to docs/overview/http-load-test-sli-guide.md diff --git a/docs/user/http-load-testing.md b/docs/overview/http-load-testing.md similarity index 100% rename from docs/user/http-load-testing.md rename to docs/overview/http-load-testing.md diff --git a/docs/user/http-soak-testing.md b/docs/overview/http-soak-testing.md similarity index 100% rename from docs/user/http-soak-testing.md rename to docs/overview/http-soak-testing.md diff --git a/docs/user/image-mirroring.md b/docs/overview/image-mirroring.md similarity index 92% rename from docs/user/image-mirroring.md rename to docs/overview/image-mirroring.md index 01a4e6e60a..2eabcf2ac6 100644 --- a/docs/user/image-mirroring.md +++ b/docs/overview/image-mirroring.md @@ -3,7 +3,7 @@ All required self-hosted NVCF artifacts (see [self-hosted-artifact-manifest](./manifest.md)) must be available to be pulled by pods in your Kubernetes cluster. The deployment bundles are `nvcf-self-managed-stack` for the control plane, `nvcf-compute-plane-stack` for the compute plane, and `nvcf-observability-stack` for standalone shared observability. This page provides examples on how to pull artifacts from NGC and push them to your desired registry. -**Mirroring images is not the same as configuring image pull secrets.** This page covers how to copy NVCF artifacts into your registry. If your registry is private, Kubernetes also needs credentials to pull those images at runtime. For instructions on configuring image pull secrets for the NVCF control plane pods, see [control-plane-image-pull-secrets](./helmfile-installation.md) in the installation guide. +**Mirroring images is not the same as configuring image pull secrets.** This page covers how to copy NVCF artifacts into your registry. If your registry is private, Kubernetes also needs credentials to pull those images at runtime. For instructions on configuring image pull secrets for the NVCF control plane pods, see [control-plane-image-pull-secrets](/nvcf/self-managed/helmfile-installation) in the installation guide. @@ -49,7 +49,7 @@ If you plan to deploy **Low Latency Streaming (LLS)**, you must mirror the follo - Streaming application images (e.g., `usd-composer`) -See [self-hosted-lls-installation](./lls-installation.md) for LLS deployment instructions. +See [self-hosted-lls-installation](/nvcf/self-managed/lls-installation) for LLS deployment instructions. ## Pulling Artifacts from NGC @@ -195,20 +195,18 @@ First, ensure you have the [NGC CLI installed and configured](https://org.ngc.nv ```bash # Set stack versions -export STACK_VERSION="0.20.7" -export COMPUTE_STACK_VERSION="0.4.4" -export OBSERVABILITY_STACK_VERSION="0.2.2" +export STACK_VERSION="1.0.0" +export COMPUTE_STACK_VERSION="1.0.0" +export OBSERVABILITY_STACK_VERSION="1.0.0" # Download a specific control-plane stack version -# Publication pending: nvcf-self-managed-stack 0.20.7 is not yet available for download. +# Publication pending: nvcf-self-managed-stack 1.0.0 is not yet available for download. # Download a specific compute-plane stack version -ngc registry resource download-version \ - "nvidia/nvcf/nvcf-compute-plane-stack:${COMPUTE_STACK_VERSION}" +# Publication pending: nvcf-compute-plane-stack 1.0.0 is not yet available for download. # Download a specific observability stack version -ngc registry resource download-version \ - "nvidia/nvcf/nvcf-observability-stack:${OBSERVABILITY_STACK_VERSION}" +# Publication pending: nvcf-observability-stack 1.0.0 is not yet available for download. ``` {/*docs-version-sync:END image-mirroring-resource-examples*/} @@ -228,7 +226,7 @@ and its listed artifact versions are QA-qualified together. {/*docs-version-sync:BEGIN image-mirroring-stack-snippet*/} ```bash -# Publication pending: nvcf-self-managed-stack 0.20.7 is not yet available for download. +# Publication pending: nvcf-self-managed-stack 1.0.0 is not yet available for download. ``` {/*docs-version-sync:END image-mirroring-stack-snippet*/} @@ -253,13 +251,7 @@ Download and extract: {/*docs-version-sync:BEGIN image-mirroring-compute-stack-snippet*/} ```bash -# Set the version -export COMPUTE_VERSION="0.4.4" - -ngc registry resource download-version "nvidia/nvcf/nvcf-compute-plane-stack:${COMPUTE_VERSION}" && \ - mkdir -p nvcf-compute-plane-stack && \ - tar -xzf nvcf-compute-plane-stack_v${COMPUTE_VERSION}/nvcf-compute-plane-stack-${COMPUTE_VERSION}.tar.gz -C nvcf-compute-plane-stack && \ - rm -rf nvcf-compute-plane-stack_v${COMPUTE_VERSION} +# Publication pending: nvcf-compute-plane-stack 1.0.0 is not yet available for download. ``` {/*docs-version-sync:END image-mirroring-compute-stack-snippet*/} @@ -287,13 +279,7 @@ Download and extract: {/*docs-version-sync:BEGIN image-mirroring-observability-stack-snippet*/} ```bash -# Set the version -export OBSERVABILITY_VERSION="0.2.2" - -ngc registry resource download-version "nvidia/nvcf/nvcf-observability-stack:${OBSERVABILITY_VERSION}" && \ - mkdir -p nvcf-observability-stack && \ - tar -xzf nvcf-observability-stack_v${OBSERVABILITY_VERSION}/nvcf-observability-stack-${OBSERVABILITY_VERSION}.tar.gz -C nvcf-observability-stack && \ - rm -rf nvcf-observability-stack_v${OBSERVABILITY_VERSION} +# Publication pending: nvcf-observability-stack 1.0.0 is not yet available for download. ``` {/*docs-version-sync:END image-mirroring-observability-stack-snippet*/} @@ -340,7 +326,7 @@ The extracted directory contains: {/*docs-version-sync:END image-mirroring-cli-snippet*/} -See [self-hosted-cli](./cli.md) for detailed configuration instructions +See [self-hosted-cli](/nvcf/self-managed/cli) for detailed configuration instructions If you don't have access to this repository, contact your NVIDIA representative. diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/apipersonalkey.png b/docs/overview/images/apipersonalkey.png similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/apipersonalkey.png rename to docs/overview/images/apipersonalkey.png diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/grpc-reconnect-flow.png b/docs/overview/images/grpc-reconnect-flow.png similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/grpc-reconnect-flow.png rename to docs/overview/images/grpc-reconnect-flow.png diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/grpc-single-client.png b/docs/overview/images/grpc-single-client.png similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/grpc-single-client.png rename to docs/overview/images/grpc-single-client.png diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/lls-self-hosted-arch-streaming-proxy.png b/docs/overview/images/lls-self-hosted-arch-streaming-proxy.png similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/lls-self-hosted-arch-streaming-proxy.png rename to docs/overview/images/lls-self-hosted-arch-streaming-proxy.png diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/low-latency-streaming.png b/docs/overview/images/low-latency-streaming.png similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/low-latency-streaming.png rename to docs/overview/images/low-latency-streaming.png diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-banner.svg b/docs/overview/images/nvcf-banner.svg similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-banner.svg rename to docs/overview/images/nvcf-banner.svg diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-gpu-cluster-targeting.svg b/docs/overview/images/nvcf-gpu-cluster-targeting.svg similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-gpu-cluster-targeting.svg rename to docs/overview/images/nvcf-gpu-cluster-targeting.svg diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-grpc-invocation-path.svg b/docs/overview/images/nvcf-grpc-invocation-path.svg similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-grpc-invocation-path.svg rename to docs/overview/images/nvcf-grpc-invocation-path.svg diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-grpc-multicluster-invocation.svg b/docs/overview/images/nvcf-grpc-multicluster-invocation.svg similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-grpc-multicluster-invocation.svg rename to docs/overview/images/nvcf-grpc-multicluster-invocation.svg diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-high-level-stack.svg b/docs/overview/images/nvcf-high-level-stack.svg similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-high-level-stack.svg rename to docs/overview/images/nvcf-high-level-stack.svg diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-http-invocation-path.svg b/docs/overview/images/nvcf-http-invocation-path.svg similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-http-invocation-path.svg rename to docs/overview/images/nvcf-http-invocation-path.svg diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-http-multicluster-invocation.svg b/docs/overview/images/nvcf-http-multicluster-invocation.svg similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-http-multicluster-invocation.svg rename to docs/overview/images/nvcf-http-multicluster-invocation.svg diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-llm-invocation-path.svg b/docs/overview/images/nvcf-llm-invocation-path.svg similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-llm-invocation-path.svg rename to docs/overview/images/nvcf-llm-invocation-path.svg diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-llm-multicluster-invocation.svg b/docs/overview/images/nvcf-llm-multicluster-invocation.svg similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-llm-multicluster-invocation.svg rename to docs/overview/images/nvcf-llm-multicluster-invocation.svg diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-lls-session.png b/docs/overview/images/nvcf-lls-session.png similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-lls-session.png rename to docs/overview/images/nvcf-lls-session.png diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-multi-region-multi-cluster.svg b/docs/overview/images/nvcf-multi-region-multi-cluster.svg similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/nvcf-multi-region-multi-cluster.svg rename to docs/overview/images/nvcf-multi-region-multi-cluster.svg diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/self-hosted-min-topology.png b/docs/overview/images/self-hosted-min-topology.png similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/images/self-hosted-min-topology.png rename to docs/overview/images/self-hosted-min-topology.png diff --git a/docs/overview/index.md b/docs/overview/index.md new file mode 100644 index 0000000000..f96600d047 --- /dev/null +++ b/docs/overview/index.md @@ -0,0 +1,30 @@ +# NVIDIA Cloud Functions + +![NVIDIA Cloud Functions banner](images/nvcf-banner.svg) + +This guide provides information for deploying and operating NVCF in self-managed environments. + +NVCF ships as three independently versioned Helm stacks. Each stack has its +own documentation set and version menu. + +- [Self-Managed Stack](/nvcf/self-managed/) + : Control plane installation, configuration, function APIs, and operations. +- [Compute Plane Stack](/nvcf/compute-plane/) + : GPU cluster setup, scheduling, and caches. +- [Observability Stack](/nvcf/observability/) + : Metrics, dashboards, and alerting. +- [Compatibility Matrix](./compatibility-matrix.md) + : Stack releases that are qualified to run together. + +## Getting started + +- [Quickstart](./quickstart.md) + : Install the control plane, register a GPU cluster, and validate the deployment with the one-click CLI flow. +- [Deployment](/nvcf/self-managed/installation-overview) + : Compare the one-click and Helmfile installation paths. +- [GPU Cluster Setup](/nvcf/compute-plane/gpu-cluster-setup) + : Connect GPU clusters to the NVCF control plane. +- [Configuration](/nvcf/self-managed/gateway-routing) + : Configure gateway routing, registries, and invocation options. +- [Using Cloud Functions](./api.md) + : Create and invoke functions using the NVCF API and CLI. diff --git a/docs/user/infrastructure-sizing.md b/docs/overview/infrastructure-sizing.md similarity index 91% rename from docs/user/infrastructure-sizing.md rename to docs/overview/infrastructure-sizing.md index 1d4c83aa10..0da60098de 100644 --- a/docs/user/infrastructure-sizing.md +++ b/docs/overview/infrastructure-sizing.md @@ -7,8 +7,8 @@ types and storage classes are AWS examples. For another cloud service provider storage classes that provide at least the listed vCPU, memory, storage, zone spread, and GPU compatibility characteristics. Actual requirements depend on workload characteristics, function count, and request concurrency. Use the -[self-managed-grpc-load-test](../g-rpc-load-testing) and -[self-managed-http-load-test](../http-load-testing) guides to validate +[self-managed-grpc-load-test](/nvcf/self-managed/g-rpc-load-testing) and +[self-managed-http-load-test](/nvcf/self-managed/http-load-testing) guides to validate throughput and tune your control plane accordingly. @@ -153,7 +153,7 @@ GPU requirements: - Physical GPU hardware on worker nodes For development and testing environments without GPUs, install the fake GPU -operator to simulate GPU resources. See [fake-gpu-operator](../fake-gpu-operator) for +operator to simulate GPU resources. See [fake-gpu-operator](/nvcf/compute-plane/fake-gpu-operator) for instructions. ## Storage Recommendations @@ -166,7 +166,7 @@ instructions. | Control Plane Services | 1 to 10 Gi each | Defaults are typically sufficient | Storage sizes are configurable via the `storageSize` value in your environment -file. See [helmfile-installation](./helmfile-installation.md) for details. +file. See [helmfile-installation](/nvcf/self-managed/helmfile-installation) for details. Some cloud providers have minimum PVC size requirements. For example, AWS EBS @@ -180,18 +180,18 @@ The default control-plane resource sizing shipped with the helmfile stack is designed to handle approximately 100 concurrent users. If you need higher throughput: -1. Benchmark your deployment using the [self-managed-grpc-load-test](../g-rpc-load-testing) - or [self-managed-http-load-test](../http-load-testing) guide. Start with +1. Benchmark your deployment using the [self-managed-grpc-load-test](/nvcf/self-managed/g-rpc-load-testing) + or [self-managed-http-load-test](/nvcf/self-managed/http-load-testing) guide. Start with `--vus 100` and increase gradually. 2. Scale node pools independently. Cassandra, OpenBao, and control-plane pools can each be scaled without affecting the others. 3. Increase pod resources for specific services by adding `values:` blocks - in the helmfile release definitions. See [helmfile-installation](./helmfile-installation.md) + in the helmfile release definitions. See [helmfile-installation](/nvcf/self-managed/helmfile-installation) for override examples. - [Quickstart](./quickstart.md): One-click fresh installation walkthrough -- [self-managed-grpc-load-test](../g-rpc-load-testing): Validate control-plane throughput -- [self-managed-http-load-test](../http-load-testing): Validate HTTP invocation throughput +- [self-managed-grpc-load-test](/nvcf/self-managed/g-rpc-load-testing): Validate control-plane throughput +- [self-managed-http-load-test](/nvcf/self-managed/http-load-testing): Validate HTTP invocation throughput diff --git a/docs/user/llm-gateway.md b/docs/overview/llm-gateway.md similarity index 99% rename from docs/user/llm-gateway.md rename to docs/overview/llm-gateway.md index 4e07be33d3..b8f25d94d6 100644 --- a/docs/user/llm-gateway.md +++ b/docs/overview/llm-gateway.md @@ -95,7 +95,7 @@ gateway does not wrap it in a second NVCF envelope. `pulsar`, or `random` for `llmConfig.routingMethod`. For the mapping to Stargate algorithms and the request-router allowlist, see -[LLM Request Router Load Balancing](./llm-request-router-load-balancing.md). +[LLM Request Router Load Balancing](/nvcf/self-managed/llm-request-router-load-balancing). `llmConfig.tokenRateLimit` applies a per-model token limit. Use one or more comma-separated limits in `-` format, where `` is a positive integer and `` is one of `S` (seconds), `M` (minutes), `H` (hours), `D` (days), or `W` (weeks). A single limit is one token budget over one time window, such as `1000-S`. A combined limit is multiple token budgets over distinct time windows, such as `1000-S,5000-M,100000-H,500000-D,1000000-W`; do not repeat a unit in the same value. diff --git a/docs/user/local-development.md b/docs/overview/local-development.md similarity index 100% rename from docs/user/local-development.md rename to docs/overview/local-development.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development/multi-cluster-cli.md b/docs/overview/local-development/multi-cluster-cli.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development/multi-cluster-cli.md rename to docs/overview/local-development/multi-cluster-cli.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development/multi-cluster-helmfile.md b/docs/overview/local-development/multi-cluster-helmfile.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development/multi-cluster-helmfile.md rename to docs/overview/local-development/multi-cluster-helmfile.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development/single-cluster-cli.md b/docs/overview/local-development/single-cluster-cli.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development/single-cluster-cli.md rename to docs/overview/local-development/single-cluster-cli.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development/single-cluster-helmfile.md b/docs/overview/local-development/single-cluster-helmfile.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development/single-cluster-helmfile.md rename to docs/overview/local-development/single-cluster-helmfile.md diff --git a/docs/user/manifest.md b/docs/overview/manifest.md similarity index 94% rename from docs/user/manifest.md rename to docs/overview/manifest.md index 31dc6e3f3f..cf5d8d36ce 100644 --- a/docs/user/manifest.md +++ b/docs/overview/manifest.md @@ -10,10 +10,12 @@ NVCF deployment. Artifacts are grouped by deployment plane and type. Artifact version compatibility -Newer artifact versions might be available. NVCF self-managed, compute-plane, -and observability stack releases are QA-qualified as umbrella releases with -the specific versions shown on this page. Use these versions together. NVIDIA -cannot guarantee compatibility when you substitute other artifact versions. +Newer artifact versions might be available. Each NVCF stack release is +QA-qualified with the artifact versions shown on this page for that stack. +Use the [compatibility matrix](/nvcf/overview/compatibility-matrix) to choose +self-managed, compute-plane, and observability releases that are qualified to +run together. NVIDIA cannot guarantee compatibility when you substitute other +artifact versions. @@ -141,15 +143,13 @@ The following tables list the complete artifact inventory. {/*docs-version-sync:BEGIN manifest-artifact-registry-paths*/} -### Stack release set - -Documentation: `dev` (development) +### Stack releases | Stack | Version | Source tag | | --- | --- | --- | -| Control plane | `0.20.7` | `deploy/stacks/self-managed/v0.20.7` | -| Compute plane | `0.4.4` | `deploy/stacks/nvcf-compute-plane/v0.4.4` | -| Observability | `0.2.2` | `deploy/stacks/observability/v0.2.2` | +| [Self-managed (control plane)](/nvcf/self-managed/) | `1.0.0` | `deploy/stacks/self-managed/v1.0.0` | +| [Compute plane](/nvcf/compute-plane/) | `1.0.0` | `deploy/stacks/nvcf-compute-plane/v1.0.0` | +| [Observability](/nvcf/observability/) | `1.0.0` | `deploy/stacks/observability/v1.0.0` | ### Control plane Helm charts @@ -174,7 +174,7 @@ Documentation: `dev` (development) | `helm-nvcf-pki` | `0.1.0` | `self-managed` | Optional | Provisions the OpenBao-backed ClusterIssuer for NVCF service TLS. | `https://helm.ngc.nvidia.com/nvidia/nvcf/helm-nvcf-pki:0.1.0` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nvcf-pki) | | `helm-nvcf-rate-limiter` | `1.2.1` | `self-managed` | Required | Deploys request rate limiting for supported invocation paths. | `https://helm.ngc.nvidia.com/nvidia/nvcf/helm-nvcf-rate-limiter:1.2.1` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/ratelimiter) | | `helm-nvcf-sis` | `2.4.0` | `self-managed` | Required | Deploys the Spot Instance Service. | `https://helm.ngc.nvidia.com/nvidia/nvcf/helm-nvcf-sis:2.4.0` | | -| `helm-nvcf-state-metrics` | `1.0.6` | `self-managed` | Required | Deploys NVCF state metrics for observability. | `Publication pending` | | +| `helm-nvcf-state-metrics` | `1.0.2` | `self-managed` | Required | Deploys NVCF state metrics for observability. | `Publication pending` | | | `helm-nvcf-ui` | `1.1.2` | `self-managed` | Optional | Deploys the optional NVCF UI admin panel. | `https://helm.ngc.nvidia.com/nvidia/nvcf/helm-nvcf-ui:1.1.2` | | | `helm-nvcf-vanity-gateway` | `0.5.0` | `self-managed` | Optional | Deploys the optional vanity hostname gateway. | `https://helm.ngc.nvidia.com/nvidia/nvcf/helm-nvcf-vanity-gateway:0.5.0` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/vanity-gateway) | | `helm-reval` | `1.4.1` | `self-managed` | Required | Deploys the function revalidation service. | `https://helm.ngc.nvidia.com/nvidia/nvcf/helm-reval:1.4.1` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/helm-reval) | @@ -201,6 +201,7 @@ Documentation: `dev` (development) | `nats-server` | `2.14.6-alpine3.22` | `self-managed` | Required | Provides messaging for function deployment and invocation. | `nvcr.io/nvidia/nvcf/nats-server:2.14.6-alpine3.22` | [Upstream](https://github.com/nats-io/nats-server) | | `nvcf-ai-api-gateway-service` | `1.35.1` | `self-managed` | Optional | Serves the optional vanity hostname gateway. | `nvcr.io/nvidia/nvcf/nvcf-ai-api-gateway-service:1.35.1` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/vanity-gateway) | | `nvcf-api-keys-service` | `1.9.1` | `self-managed` | Required | Creates and manages NVCF API keys. | `nvcr.io/nvidia/nvcf/nvcf-api-keys-service:1.9.1` | | +| `nvcf-cassandra-migrations` | `0.17.6` | `self-managed` | Required | Applies the Cassandra schemas required by NVCF services. | `nvcr.io/nvidia/nvcf/nvcf-cassandra-migrations:0.17.6` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/migrations/cassandra) | | `nvcf-ess` | `0.5.0` | `self-managed` | Required | Provides encrypted application secrets to NVCF workloads. | `nvcr.io/nvidia/nvcf/nvcf-ess:0.5.0` | | | `nvcf-function-autoscaler` | `1.21.8` | `self-managed` | Required | Scales functions from NVCF workload metrics. | `nvcr.io/nvidia/nvcf/nvcf-function-autoscaler:1.21.8` | | | `nvcf-grpc-proxy` | `1.33.5` | `self-managed` | Required | Proxies bidirectional gRPC traffic between the control and compute planes. | `nvcr.io/nvidia/nvcf/nvcf-grpc-proxy:1.33.5` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/grpc-proxy) | @@ -211,7 +212,7 @@ Documentation: `dev` (development) | `nvcf-openbao-migrations` | `0.19.5` | `self-managed` | Required | Applies the OpenBao configuration required by NVCF. | `nvcr.io/nvidia/nvcf/nvcf-openbao-migrations:0.19.5` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/migrations/openbao) | | `nvcf-ratelimiter` | `1.17.3` | `self-managed` | Required | Enforces request rate limits for supported invocation paths. | `nvcr.io/nvidia/nvcf/nvcf-ratelimiter:1.17.3` | | | `nvcf-service-oss` | `1.18.0` | `self-managed` | Required | Provides the primary NVCF control-plane API. | `nvcr.io/nvidia/nvcf/nvcf-service-oss:1.18.0` | | -| `nvcf-state-metrics-service` | `1.24.1` | `self-managed` | Required | Exports NVCF resource state as Prometheus metrics. | `Publication pending` | | +| `nvcf-state-metrics-service` | `1.23.7` | `self-managed` | Required | Exports NVCF resource state as Prometheus metrics. | `Publication pending` | | | `nvcf-ui` | `1.1.2` | `self-managed` | Optional | Serves the optional NVCF administrative interface. | `nvcr.io/nvidia/nvcf/nvcf-ui:1.1.2` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/uis/nvcf-ui) | | `nvcf-worker-init-oss` | `1.2.1` | `self-managed` | Required | Prepares function resources before the user container starts. | `nvcr.io/nvidia/nvcf/nvcf-worker-init-oss:1.2.1` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/worker-init) | | `nvcf-worker-llm-credentials-oss` | `1.1.2` | `self-managed` | Required | Maintains a current NVCF worker token for LLM function workloads. | `nvcr.io/nvidia/nvcf/nvcf-worker-llm-credentials-oss:1.1.2` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/worker-llm-credentials) | @@ -285,21 +286,13 @@ Documentation: `dev` (development) | --- | --- | --- | --- | --- | --- | --- | | `nats-server-config-reloader` | `0.24.0` | `compute-plane` / `self-managed` | Required | Reloads NATS configuration for the control plane and optional NVIDIA Dynamo deployment. | `docker.io/natsio/nats-server-config-reloader:0.24.0` | [Upstream](https://github.com/nats-io/k8s) | -### EA-only CVE-impacted artifacts - -These Early Access artifacts have known CVE impact. Use only the QA-qualified versions listed for this EA stack. - -| Artifact | Version | Stack | Required | Description | Distribution | Source code | -| --- | --- | --- | --- | --- | --- | --- | -| `nvcf-cassandra-migrations` | `0.17.6` | `self-managed` | Required | Applies the Cassandra schemas required by Early Access NVCF services. | `nvcr.io/nvidia/nvcf/nvcf-cassandra-migrations:0.17.6` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/migrations/cassandra) | - ### Tools and deployment resources | Artifact | Version | Stack | Description | Distribution | Source code | | --- | --- | --- | --- | --- | --- | | `nvcf-cli` | `1.16.2` | Independent | Manages functions, deployments, and clusters from the command line. | `nvcr.io/nvidia/nvcf/nvcf-cli:1.16.2` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/src/clis/nvcf-cli) | -| `nvcf-compute-plane-stack` | `0.4.4` | `compute-plane` | Provides the Helmfile bundle for compute-plane deployment. | `nvcr.io/nvidia/nvcf/nvcf-compute-plane-stack:0.4.4` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/nvcf-compute-plane) | -| `nvcf-observability-stack` | `0.2.2` | `observability` | Provides the Helmfile bundle for standalone observability deployment. | `nvcr.io/nvidia/nvcf/nvcf-observability-stack:0.2.2` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/observability) | -| `nvcf-self-managed-stack` | `0.20.7` | `self-managed` | Provides the Helmfile bundle for control-plane deployment. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/self-managed) | +| `nvcf-compute-plane-stack` | `1.0.0` | `compute-plane` | Provides the Helmfile bundle for compute-plane deployment. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/nvcf-compute-plane) | +| `nvcf-observability-stack` | `1.0.0` | `observability` | Provides the Helmfile bundle for standalone observability deployment. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/observability) | +| `nvcf-self-managed-stack` | `1.0.0` | `self-managed` | Provides the Helmfile bundle for control-plane deployment. | `Publication pending` | [GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/self-managed) | {/*docs-version-sync:END manifest-artifact-registry-paths*/} diff --git a/docs/user/cluster-management/multi-tenancy.md b/docs/overview/multi-tenancy.md similarity index 88% rename from docs/user/cluster-management/multi-tenancy.md rename to docs/overview/multi-tenancy.md index 0bd29c8a5f..b2d63d7685 100644 --- a/docs/user/cluster-management/multi-tenancy.md +++ b/docs/overview/multi-tenancy.md @@ -17,7 +17,7 @@ A function is deployed with a private or access-controlled endpoint. Each invoki **Characteristics:** - Helm-deployed functions run in a dedicated Kubernetes namespace, isolated from other functions at the namespace boundary. Custom container functions run in the shared `nvcf-backend` namespace and do not receive namespace-level isolation. -- When [Account Isolation](./configuration.md#account-isolated-clusters) is enabled on the cluster, workload pods are scheduled on nodes exclusively reserved for that NCA account — no other account's pods share those nodes. +- When [Account Isolation](/nvcf/compute-plane/cluster-configuration#account-isolated-clusters) is enabled on the cluster, workload pods are scheduled on nodes exclusively reserved for that NCA account — no other account's pods share those nodes. - Network policies restrict pod-to-pod communication across namespaces (subject to the cluster's CNI supporting Kubernetes NetworkPolicy). **Best for:** Tenants with strict data separation requirements who can accept higher resource cost in exchange for stronger isolation. @@ -45,10 +45,10 @@ The table below summarizes the isolation available at each layer and how to enab | Layer | Default Behavior | Stronger Option | How to Enable | |---|---|---|---| | **Namespace** | Helm-deployed functions run in a dedicated Kubernetes namespace; custom container functions run in the shared `nvcf-backend` namespace | — | Namespace isolation is automatic for helm deployments; not available for custom container deployments | -| **Network** | Egress and cross-namespace traffic controlled by NetworkPolicy | Custom policies via `nvca-namespace-networkpolicies` configmap | See [Network Configuration](./configuration.md#network-configuration) | -| **Node (function-level)** | Functions and tasks may share nodes | One active workload instance per node via k8s scheduling constraints | Enable `HostIsolation` cluster attribute — see [Host-Isolated Clusters](./configuration.md#host-isolated-clusters) | -| **Node (account-level)** | Multiple NCA accounts may share nodes | Hard anti-affinity: pods from different NCA accounts are never co-located via k8s scheduling constraints | Enable `AccountIsolation` cluster attribute — see [Account-Isolated Clusters](./configuration.md#account-isolated-clusters) | -| **Container runtime** | Standard container runtime (runc) | Kata Containers — each pod runs inside a lightweight VM | Enable `KataRuntimeIsolation` cluster attribute — see [Kata Container-Isolated Workloads](./configuration.md#kata-container-isolated-workloads) | +| **Network** | Egress and cross-namespace traffic controlled by NetworkPolicy | Custom policies via `nvca-namespace-networkpolicies` configmap | See [Network Configuration](/nvcf/compute-plane/cluster-configuration#network-configuration) | +| **Node (function-level)** | Functions and tasks may share nodes | One active workload instance per node via k8s scheduling constraints | Enable `HostIsolation` cluster attribute — see [Host-Isolated Clusters](/nvcf/compute-plane/cluster-configuration#host-isolated-clusters) | +| **Node (account-level)** | Multiple NCA accounts may share nodes | Hard anti-affinity: pods from different NCA accounts are never co-located via k8s scheduling constraints | Enable `AccountIsolation` cluster attribute — see [Account-Isolated Clusters](/nvcf/compute-plane/cluster-configuration#account-isolated-clusters) | +| **Container runtime** | Standard container runtime (runc) | Kata Containers — each pod runs inside a lightweight VM | Enable `KataRuntimeIsolation` cluster attribute — see [Kata Container-Isolated Workloads](/nvcf/compute-plane/cluster-configuration#kata-container-isolated-workloads) | | **Hardware (GPU)** | Shared GPU; no hardware isolation | **Not supported by NVCF** — hardware-level GPU isolation is the infrastructure provider's responsibility | — | diff --git a/docs/user/quickstart.md b/docs/overview/quickstart.md similarity index 92% rename from docs/user/quickstart.md rename to docs/overview/quickstart.md index 3c0dfec16c..4a924d4c68 100644 --- a/docs/user/quickstart.md +++ b/docs/overview/quickstart.md @@ -7,8 +7,8 @@ healthy. This quickstart uses a single k3d cluster named `ncp-local`, fake GPUs, and local route hostnames. It is for local development and validation only. For a remote deployment, or for separate control-plane and GPU clusters, use -[Helmfile Installation](./helmfile-installation.md) and -[Self-Managed Clusters](./cluster-management/self-managed.md). +[Helmfile Installation](/nvcf/self-managed/helmfile-installation) and +[Self-Managed Clusters](/nvcf/compute-plane/self-managed-clusters). Run the commands from the NVCF repository root unless a step says otherwise. The `nvcf-cli self-hosted up` command runs on your workstation. It does not run @@ -31,7 +31,7 @@ Before you start, install and prepare: - `helm` >= 3.14 - `helmfile` >= 1.0. Use `helmfile` >= 1.5.0 with Helm 4. - `helm-diff` plugin -- `nvcf-cli` on your `PATH`. See [Installation](./cli.md#installation) to +- `nvcf-cli` on your `PATH`. See [Installation](/nvcf/self-managed/cli#installation) to build it from the repository or download it from NGC. - An NGC API key with access to the NVCF chart and image registry - The NGC organization and team slugs for that registry access @@ -101,7 +101,7 @@ export PATH="${HOME}/.local/bin:${PATH}" nvcf-cli version ``` -For the packaged CLI release, see [Installation](./cli.md#download-from-ngc). +For the packaged CLI release, see [Installation](/nvcf/self-managed/cli#download-from-ngc). `self-hosted up` defaults to `--env local` and supports only the single local @@ -321,7 +321,7 @@ make -C tools/ncp-local-cluster destroy ## Troubleshooting -If the quickstart fails, start with [Troubleshooting](./troubleshooting.md). +If the quickstart fails, start with [Troubleshooting](/nvcf/self-managed/troubleshooting). Common local k3d issues: @@ -343,11 +343,11 @@ Common local k3d issues: ``` For non-local clusters, see - [Node inotify limits](./cluster-management/self-managed.md#node-inotify-limits). + [Node inotify limits](/nvcf/compute-plane/self-managed-clusters#node-inotify-limits). ## See Also - [Local Development](../local-development) for local k3d variants and cleanup commands. -- [Helmfile Installation](./helmfile-installation.md) for remote or manual control-plane installs. -- [Self-Managed Clusters](./cluster-management/self-managed.md) for registering GPU clusters outside the local quickstart. +- [Helmfile Installation](/nvcf/self-managed/helmfile-installation) for remote or manual control-plane installs. +- [Self-Managed Clusters](/nvcf/compute-plane/self-managed-clusters) for registering GPU clusters outside the local quickstart. - `src/clis/nvcf-cli/examples/` in this repository for sample CLI input files. diff --git a/docs/overview/release-notes/0.5.0-to-0.6.0-upgrade.md b/docs/overview/release-notes/0.5.0-to-0.6.0-upgrade.md new file mode 100644 index 0000000000..60c863f619 --- /dev/null +++ b/docs/overview/release-notes/0.5.0-to-0.6.0-upgrade.md @@ -0,0 +1,474 @@ +# 0.5.0 to 0.6.0 Upgrade + +The upgrade from 0.5.0 to 0.6.0 crosses two scheduled data migrations and one +required Spot Instance Service cleanup boundary. Run Stops 1 and 2, sync the full 0.6.0 stack +while the Spot Instance Service migration runtime can restore the global NATS streams, and then +run Stop 3. + + +These steps assume 0.5.0 is already deployed and healthy, that you have downloaded the 0.6.0 stack, and that you run `make` from the extracted `nvcf-self-managed-stack/` directory with your environment set. + + +```bash +export HELMFILE_ENV= # for example eks-cds-qa +``` + +## Prerequisite: upgrade to 0.5.1 or later first + + +If you pull NVCF images from a private registry (for example NGC `nvcr.io`) and rely on `global.imagePullSecrets` to deliver the pull secret, move to 0.5.1 or later before you start this procedure. Release 0.5.0 does not propagate `global.imagePullSecrets` to the chart pods, so private migration and sidecar images (for example `nvcf-cassandra-migrations` and `nats-box`) fail to pull with `ImagePullBackOff` / `403`, and a clean 0.5.0 install never becomes healthy. 0.5.1 adds that propagation. If you already run 0.5.1 or later, skip this section. + + +Moving from 0.5.0 to 0.5.1 is a normal patch sync, not a migration stop. Download the 0.5.1 stack and sync it with your existing environment: + +```bash +# Create the pull secret in every NVCF namespace (private registry only). +# The 0.5.1 charts reference this secret by name; without it there is nothing to propagate. +kubectl create namespace cert-manager \ + --dry-run=client -o yaml | kubectl apply -f - + +for ns in cassandra-system nats-system nvcf api-keys ess sis vault-system nvca-operator nvca-system nvcf-backend cert-manager; do + kubectl create secret docker-registry nvcr-pull-secret \ + --docker-server= --docker-username='' --docker-password='' \ + -n "$ns" --dry-run=client -o yaml | kubectl apply -f - +done + +# Confirm the env references the secret so every chart inherits it. +# environments/.yaml should contain: +# global: +# imagePullSecrets: +# - name: nvcr-pull-secret + +# Sync the full 0.5.1 stack so every chart re-renders with imagePullSecrets set. +make install HELMFILE_ENV="$HELMFILE_ENV" + +# Confirm the control plane is healthy before starting the 0.6.0 upgrade. +# This should print nothing; any output is a pod that is not yet Running/Completed. +kubectl get pods -A | grep -vE 'Running|Completed|READY' +``` + +## Prerequisite: install nvcf-cli 1.8.3 or later + +Use `nvcf-cli` 1.8.3 or later when you re-register GPU clusters after the control-plane upgrade. Earlier versions return the existing cluster identity without refreshing its OIDC issuer and JSON Web Key Set (JWKS). A cluster upgraded from 0.5.x can therefore retain an empty JWKS and the NVCA agent cannot authenticate. + +Confirm the CLI version before starting the upgrade: + +```bash +nvcf-cli version +``` + +## Prepare the 0.6.0 configuration + +Return to the extracted 0.6.0 stack before running any 0.6.0 sync command. Reconcile the site-specific settings from the 0.5.1 deployment into the 0.6.0 configuration files: + +- `environments/$HELMFILE_ENV.yaml` +- `secrets/$HELMFILE_ENV-secrets.yaml` + +Start from the files or templates included with 0.6.0 and carry forward the required values from 0.5.1. Do not replace the 0.6.0 files wholesale because the available settings can change between releases. Preserve the existing registry, storage, endpoint, credential, and secret values unless this procedure explicitly instructs you to change them. + +Confirm that both files exist before continuing: + +```bash +for file in \ + "environments/${HELMFILE_ENV}.yaml" \ + "secrets/${HELMFILE_ENV}-secrets.yaml"; do + [ -f "$file" ] || { echo "missing required file: $file"; exit 1; } +done +``` + +### Enable OpenBao issuer discovery for CSP clusters + +Cloud service provider (CSP) clusters such as Amazon EKS use an external OpenID Connect (OIDC) issuer for Kubernetes service account tokens. Enable issuer discovery in `environments/$HELMFILE_ENV.yaml` so the OpenBao migration configures the external issuer's JWKS URL: + +```yaml +openbao: + migrations: + issuerDiscovery: + enabled: true +``` + + +Do not continue with the 0.6.0 sync on a CSP cluster while issuer discovery is disabled. The OpenBao migration otherwise falls back to a Kubernetes API server public key that cannot validate tokens signed by an external CSP issuer. New vault-agent logins then fail and prevent service pods from starting. + + +### Enable NATS Gateway routing + +NVCA connects to NATS through the external Gateway on TCP port `4222`. A +Gateway created with an earlier version of the quickstart might not have the +required listener. Set the Gateway name and namespace used by your environment, +label the route-owning namespace, and add the listener if it is missing: + +```bash +export NATS_GATEWAY_NAME=nvcf-gateway +export NATS_GATEWAY_NAMESPACE=envoy-gateway + +kubectl label namespace "$NATS_GATEWAY_NAMESPACE" nvcf/platform=true --overwrite + +if [ "$(kubectl -n "$NATS_GATEWAY_NAMESPACE" get gateway "$NATS_GATEWAY_NAME" \ + -o jsonpath='{.spec.listeners[?(@.name=="nats")].name}')" != "nats" ]; then + kubectl -n "$NATS_GATEWAY_NAMESPACE" patch gateway "$NATS_GATEWAY_NAME" \ + --type=json \ + -p='[{"op":"add","path":"/spec/listeners/-","value":{"name":"nats","protocol":"TCP","port":4222,"allowedRoutes":{"namespaces":{"from":"Selector","selector":{"matchLabels":{"nvcf/platform":"true"}}}}}}]' +fi +``` + +Enable the NATS route in `environments/$HELMFILE_ENV.yaml`. Use the same +Gateway name and namespace as the preceding command: + +```yaml +ingress: + gatewayApi: + routes: + nats: + enabled: true + gateways: + nats: + name: nvcf-gateway + namespace: envoy-gateway +``` + +Sync the ingress release so it creates the NATS `TCPRoute`: + +```bash +make install HELMFILE_ENV="$HELMFILE_ENV" HELMFILE_SELECTOR=name=ingress +``` + +Confirm that the listener and route exist and that the Envoy service exposes +port `4222` before continuing: + +```bash +kubectl -n "$NATS_GATEWAY_NAMESPACE" get gateway "$NATS_GATEWAY_NAME" \ + -o jsonpath='{.spec.listeners[?(@.name=="nats")].port}{"\n"}' +kubectl -n "$NATS_GATEWAY_NAMESPACE" get tcproute nats +kubectl get service -n envoy-gateway-system \ + -l "gateway.envoyproxy.io/owning-gateway-name=$NATS_GATEWAY_NAME" \ + -o jsonpath='{.items[0].spec.ports[?(@.port==4222)].port}{"\n"}' +``` + +Both port checks must print `4222`, and the `nats` `TCPRoute` must exist. Do not +continue to the NVCA refresh if any check fails. + +## Stop 1: Cassandra schema migration + + +Keep the Cassandra storage settings unchanged during this upgrade. `global.storageClass` and `global.storageSize` in the upgrade environment must produce the same values as the original installation. These settings render into the Cassandra StatefulSet `volumeClaimTemplates`, which Kubernetes treats as immutable. Changing either value, including introducing a value that was previously unset, causes the Cassandra Helm upgrade to fail. + +Do not use this upgrade procedure to resize Cassandra storage or change its storage class. + + +Before entering the maintenance window, record the storage class and size from the live Cassandra StatefulSet: + +```bash +kubectl -n cassandra-system get statefulset cassandra \ + -o jsonpath='{range .spec.volumeClaimTemplates[*]}{.metadata.name}{": storageClass="}{.spec.storageClassName}{", size="}{.spec.resources.requests.storage}{"\n"}{end}' +``` + +Confirm that the effective `global.storageClass` and `global.storageSize` settings for `HELMFILE_ENV` match the live values. If they differ, stop and restore the settings from the original installation before continuing. + +Quiesce the control plane so no service writes to the database while the migration runs: + +```bash +for ns in nvcf api-keys ess sis; do + kubectl -n "$ns" scale deployment --all --replicas=0 +done +``` + +Sync only the Cassandra release from the 0.6.0 stack. This runs the schema migration: + +```bash +make install HELMFILE_ENV="$HELMFILE_ENV" HELMFILE_SELECTOR=name=cassandra +``` + +The migration runs as a Helm `post-upgrade` hook, so this command blocks until the migration Job completes and fails if it does not. A non-zero exit means the migration did not finish; resolve it before continuing. Confirm the migration Job succeeded: + +```bash +kubectl get jobs -n cassandra-system +# the *-migrations Job should report COMPLETIONS 1/1 +``` + +## Stop 2: Spot Instance Service auth-client-id migration + +Stop 2 must reach Spot Instance Service runtime `1.561.1`. This is the released version of the +patched Spot Instance Service runtime that enables the auth-client-id migration task for the +self-hosted Spring profile. The task copies each existing cluster row's +`ssa_client_id` into `auth_client_id`. This runtime also enables the global NATS +stream validation task for the self-hosted Spring profile. + + +Do not deploy Spot Instance Service runtime `1.562.2` or later until this migration is complete. Those versions remove the migration task and expect `auth_client_id` to already be populated. + + +Temporarily override the Spot Instance Service runtime image in +`environments/$HELMFILE_ENV.yaml`: + +```yaml +sis: + image: + tag: 1.561.1 +``` + +Sync only the Spot Instance Service release from the 0.6.0 stack. The `make` target loads the +selected site environment and its matching secrets file: + +```bash +make install HELMFILE_ENV="$HELMFILE_ENV" HELMFILE_SELECTOR=name=sis +``` + +Confirm that the Spot Instance Service deployment uses runtime `1.561.1`: + +```bash +kubectl -n sis get deployment spot-instance-service \ + -o jsonpath='{.spec.template.spec.containers[*].image}{"\n"}' +``` + +Scale Spot Instance Service up and wait for it to become ready. Keep the other control-plane deployments stopped while the migration runs: + +```bash +kubectl -n sis scale deployment/spot-instance-service --replicas=1 +kubectl -n sis rollout status deployment/spot-instance-service --timeout=10m +``` + +Enable the auth-client-id migration task on Spot Instance Service: + + +Spot Instance Service already runs with the self-hosted Spring profile set by its chart. Do not +change `SPRING_PROFILES_ACTIVE`. The command below only adds the migration-task +variables and leaves the existing profile untouched. + + +```bash +kubectl set env deployment/spot-instance-service -n sis \ + NVCA_AUTH_CLIENT_ID_MIGRATION_TASK_ENABLED=true \ + NVCA_AUTH_CLIENT_ID_MIGRATION_TASK_LOCK_TTL_IN_SECONDS=60 \ + NVCA_AUTH_CLIENT_ID_MIGRATION_TASK_INITIAL_DELAY_DURATION=PT30S \ + NVCA_AUTH_CLIENT_ID_MIGRATION_TASK_SCHEDULE_DURATION=PT1M +kubectl -n sis rollout status deployment/spot-instance-service --timeout=10m +``` + +Wait at least two minutes for the scheduled task to run: + +```bash +sleep 120 +``` + +Inspect the cluster rows: + +```bash +CASSANDRA_PASSWORD=$(kubectl get secret -n cassandra-system cassandra \ + -o jsonpath='{.data.cassandra-password}' | base64 --decode) +kubectl exec -n cassandra-system cassandra-0 -- \ + cqlsh -u cassandra -p "$CASSANDRA_PASSWORD" \ + -e "SELECT cluster_id, ssa_client_id, auth_client_id FROM sis_api.cluster_by_cluster_id;" +``` + +For every row with a non-empty `ssa_client_id`, `auth_client_id` must contain the same value. Do not continue if any eligible row has an empty or mismatched `auth_client_id`. + +### Enable Spot Instance Service NATS reconnect + +The Spot Instance Service chart included with 0.6.0 does not enable NATS reconnect. Enable it on +the live ConfigMap before the full-stack sync rolls NATS, then restart Spot Instance Service so +the pod reads the new value: + +```bash +kubectl patch configmap spot-instance-service-env -n sis --type merge \ + -p '{"data":{"NATS_RECONNECT_ALLOWED":"true"}}' +kubectl rollout restart deployment/spot-instance-service -n sis +kubectl rollout status deployment/spot-instance-service -n sis --timeout=10m +``` + +Confirm that reconnect is enabled: + +```bash +test "$(kubectl -n sis get configmap spot-instance-service-env \ + -o jsonpath='{.data.NATS_RECONNECT_ALLOWED}')" = "true" +``` + +Do not continue if the command returns a non-zero exit code. A later Spot Instance Service chart +release manages this setting directly, but the 0.6.0 upgrade requires this +one-time patch. + +## Sync the full 0.6.0 stack + +Keep the `sis.image.tag: 1.561.1` override in place and sync the full +0.6.0 stack. This upgrades NATS while Spot Instance Service can recreate the global JetStream +streams if the NATS upgrade removes them: + +```bash +make install HELMFILE_ENV="$HELMFILE_ENV" +``` + +Confirm that Spot Instance Service remains ready after the full sync: + +```bash +kubectl -n sis rollout status deployment/spot-instance-service --timeout=10m +``` + +Confirm that the full-stack sync preserved the reconnect setting: + +```bash +test "$(kubectl -n sis get configmap spot-instance-service-env \ + -o jsonpath='{.data.NATS_RECONNECT_ALLOWED}')" = "true" || { + echo "Spot Instance Service NATS reconnect is not enabled; reapply the preceding patch" >&2 + exit 1 +} +``` + +The Spot Instance Service validation task runs every three minutes. During the NATS roll, Spot Instance Service +may log `A JetStream context can't be established during close` while its NATS +client reconnects. This error is transient during reconnection. Wait up to ten +minutes for both global streams to exist: + +```bash +NATS_BOX_POD="$(kubectl -n nats-system get pod \ + -l app.kubernetes.io/component=nats-box \ + -o jsonpath='{.items[0].metadata.name}')" + +for attempt in 0 1 2 3 4 5 6 7 8 9 10; do + if kubectl -n nats-system exec "$NATS_BOX_POD" -- \ + nats --context default stream info TerminateNvcaStream >/dev/null && \ + kubectl -n nats-system exec "$NATS_BOX_POD" -- \ + nats --context default stream info CreateNvcaFunctionTaskStream >/dev/null; then + break + fi + + if [ "$attempt" -eq 10 ]; then + echo "required global NATS streams were not recreated" >&2 + exit 1 + fi + sleep 60 +done + +for stream in TerminateNvcaStream CreateNvcaFunctionTaskStream; do + kubectl -n nats-system exec "$NATS_BOX_POD" -- \ + nats --context default stream info "$stream" +done +``` + +Do not continue unless both stream information commands succeed. + +If the streams are still absent after ten minutes, restart Spot Instance Service to reestablish +its JetStream context, wait for the rollout, and then rerun the preceding +stream check: + +```bash +kubectl rollout restart deployment/spot-instance-service -n sis +kubectl rollout status deployment/spot-instance-service -n sis --timeout=10m +``` + +## Stop 3: upgrade Spot Instance Service to the cleanup runtime + +Remove the temporary `sis.image.tag` override from +`environments/$HELMFILE_ENV.yaml`. Then sync the Spot Instance Service release again. The 0.6.0 +Spot Instance Service chart deploys runtime `1.563.1`. Helm preserves the migration-task +environment variables because Stop 2 added them directly to the live +Deployment. Remove them explicitly after the sync. + +```bash +make install HELMFILE_ENV="$HELMFILE_ENV" HELMFILE_SELECTOR=name=sis +kubectl set env deployment/spot-instance-service -n sis \ + NVCA_AUTH_CLIENT_ID_MIGRATION_TASK_ENABLED- \ + NVCA_AUTH_CLIENT_ID_MIGRATION_TASK_LOCK_TTL_IN_SECONDS- \ + NVCA_AUTH_CLIENT_ID_MIGRATION_TASK_INITIAL_DELAY_DURATION- \ + NVCA_AUTH_CLIENT_ID_MIGRATION_TASK_SCHEDULE_DURATION- +kubectl -n sis rollout status deployment/spot-instance-service --timeout=10m +``` + +Confirm that the cleanup-runtime sync preserved the reconnect setting: + +```bash +test "$(kubectl -n sis get configmap spot-instance-service-env \ + -o jsonpath='{.data.NATS_RECONNECT_ALLOWED}')" = "true" || { + echo "Spot Instance Service NATS reconnect is not enabled; reapply the preceding patch" >&2 + exit 1 +} +``` + +Confirm that Spot Instance Service now uses runtime `1.563.1`: + +```bash +kubectl -n sis get deployment spot-instance-service \ + -o jsonpath='{.spec.template.spec.containers[*].image}{"\n"}' +``` + +Confirm that the temporary migration-task variables are no longer present. The +output must not include variables whose names start with +`NVCA_AUTH_CLIENT_ID_MIGRATION_TASK_`: + +```bash +kubectl set env deployment/spot-instance-service -n sis --list +``` + + +Spot Instance Service runtime `1.563.1` does not run periodic global NATS stream validation under +the self-hosted Spring profile. If either global stream is lost after Stop 3, +temporarily restore the `1.561.1` image override, sync Spot Instance Service, repeat the +stream checks above, remove the override, and sync Spot Instance Service back to `1.563.1`. Do not +refresh the NVCA operator while either stream is missing. + + +## Refresh the GPU cluster registration and NVCA operator + +After the control plane is on 0.6.0, use the compute-plane stack bundle shipped for the 0.6.0 release to re-register each GPU cluster and reinstall the NVCA operator so the worker layer reconnects to the upgraded control plane with the migrated `nvca` auth-client identity. `nvcf-default` is the default cluster name; substitute your own where it applies. See [Self-Managed Clusters](/nvcf/self-managed/v0.6.1/self-managed-clusters) for the full registration reference. + +Download and extract the `nvcf-compute-plane-stack` bundle listed in the 0.6.0 artifact manifest. Run the remaining commands in this section from the extracted compute-plane stack directory. + + +Do not delete the existing cluster registration. `nvcf-cli` 1.8.3 or later refreshes the OIDC issuer and JWKS while preserving the existing `clusterID` and `clusterGroupID`. Deleting the registration creates a new cluster identity and can orphan existing function deployments. + + +Re-register the cluster with the upgraded control plane (idempotent): + +```bash +make register-cluster HELMFILE_ENV="$HELMFILE_ENV" CLUSTER_NAME=nvcf-default +``` + + +`make register-cluster` defaults to local development values (`CLUSTER_REGION=us-west-1`, `ICMS_URL=http://sis.localhost:8080`) and a sibling CLI checkout. On EKS or any other CSP, pass your real region, the control-plane ingress address, and your CLI build and config so registration reaches the right ICMS and writes correct values: + +Set the ICMS URL and Host header in the CLI configuration. Replace both placeholders with resolved values. Static YAML does not expand shell variables such as `$GATEWAY_ADDR`. + +```yaml +icms_url: "http://" +icms_host: "sis." +``` + +```bash +make register-cluster HELMFILE_ENV="$HELMFILE_ENV" CLUSTER_NAME=nvcf-default \ + CLUSTER_REGION= ICMS_URL=http:// \ + NVCF_CLI= NVCF_CLI_CONFIG= +``` + + + +Install the compute-plane stack from the refreshed registration values, then wait for the NVCA operator to roll out: + +```bash +make install HELMFILE_ENV="$HELMFILE_ENV" CLUSTER_NAME=nvcf-default +kubectl -n nvca-operator rollout status deployment/nvca-operator --timeout=10m +``` + +Confirm the backend reconnects and reports healthy: + +```bash +kubectl wait nvcfbackend nvcf-default -n nvca-operator \ + --for=jsonpath='{.status.agentStatus}'=healthy --timeout=10m +``` + +### Recover an NVCA agent that returns HTTP 401 + +If an earlier registration attempt left the cluster JWKS empty, the NVCA agent can repeatedly fail to register with ICMS and return HTTP 401. Rotate the JWKS for the existing cluster instead of deleting its registration: + +```bash +nvcf-cli --config cluster rotate \ + --cluster-id \ + --kubeconfig \ + --icms-url http:// \ + --force +``` + +The command discovers the current OIDC issuer and JWKS from the GPU cluster and updates the existing ICMS row. It preserves the cluster identity. Confirm that the backend becomes healthy after the agent retries: + +```bash +kubectl wait nvcfbackend nvcf-default -n nvca-operator \ + --for=jsonpath='{.status.agentStatus}'=healthy --timeout=10m +``` diff --git a/docs/overview/release-notes/0.6.0-to-0.6.1-upgrade.md b/docs/overview/release-notes/0.6.0-to-0.6.1-upgrade.md new file mode 100644 index 0000000000..b426057e04 --- /dev/null +++ b/docs/overview/release-notes/0.6.0-to-0.6.1-upgrade.md @@ -0,0 +1,441 @@ +# 0.6.0 to 0.6.1 Upgrade + +The 0.6.1 release replaces the archived Bitnami Cassandra runtime with the +official Apache Cassandra 5.0.8 image and an in-house Cassandra Helm chart. This +removes the CVE-affected Bitnami runtime and its bundled components. + +Fresh installs need no special handling. Existing clusters that already hold +data need a migration, because three things change at once: the runtime image, +the Helm chart shape, and the on-disk data layout. A normal in-place Helm +upgrade does not carry an existing Bitnami-based cluster onto the new stack: + +- The new Cassandra chart intentionally removes the `app.kubernetes.io/name` + and `app.kubernetes.io/instance` labels from + `volumeClaimTemplates.metadata.labels`. This is the only change to an + immutable StatefulSet field. Kubernetes treats the entire volume claim + template as immutable, so the StatefulSet has to be recreated. +- The Bitnami image stored data nested one directory deeper than the official + image expects (`/bitnami/cassandra/data/...` versus + `/var/lib/cassandra/...`), so the new image does not find the old data by + default. +- The old and new node configuration must agree on the settings the existing + data was written with. + +Because of this, an operator with an existing data-bearing cluster chooses a +migration method. This guide describes the methods and their trade-offs; none is +mandated over the others. + + +These steps assume 0.6.0 is already deployed and healthy, that you have +downloaded the 0.6.1 stack, and that you run `make` from the extracted +`nvcf-self-managed-stack/` directory with your environment set. + + + +For an existing data-bearing cluster, do not run an unscoped 0.6.1 stack +install before completing the Cassandra steps for your chosen migration method. +This includes `make install`, `helmfile sync`, and equivalent sync commands. An +unscoped sync attempts to patch the existing Cassandra StatefulSet, fails on its +immutable fields, leaves the Cassandra Helm release in a failed state, and stops +the later Helmfile stages. + +If an unscoped sync already failed, do not uninstall Cassandra or delete its +PersistentVolumeClaims. Continue with the selector-scoped Cassandra install in +your chosen migration method. That install replaces the failed release. Verify +the migration before running the full stack install. + + +```bash +export HELMFILE_ENV= # for example eks-cds-qa +``` + +## Prepare the 0.6.1 configuration + +Return to the extracted 0.6.1 stack before running any 0.6.1 sync command. +Reconcile the site-specific settings from the 0.6.0 deployment into the 0.6.1 +configuration files: + +- `environments/$HELMFILE_ENV.yaml` +- `secrets/$HELMFILE_ENV-secrets.yaml` + +Start from the files or templates included with 0.6.1 and carry forward the +required values from 0.6.0. Do not replace the 0.6.1 files wholesale because +available settings can change between releases. Preserve the existing registry, +storage, endpoint, credential, and secret values unless this procedure +explicitly instructs you to change them. + + +Do not change the Cassandra superuser password or application role passwords +during this migration. Preserve the Cassandra and OpenBao credential values +from the 0.6.0 deployment. The 0.6.1 Cassandra chart does not create the old +`cassandra-system/cassandra` Secret, so do not rely on that Secret after the +upgrade. Rotate Cassandra credentials in a separate maintenance procedure. + + +Confirm that both files exist before continuing: + +```bash +for file in \ + "environments/${HELMFILE_ENV}.yaml" \ + "secrets/${HELMFILE_ENV}-secrets.yaml"; do + [ -f "$file" ] || { echo "missing required file: $file"; exit 1; } +done +``` + +## Fresh installs + +A new 0.6.1 install has no existing Cassandra data and needs no migration. +Install the stack normally; Cassandra comes up on the official image with the +in-house chart: + +```bash +make install HELMFILE_ENV="$HELMFILE_ENV" +``` + +The rest of this guide applies only when migrating an existing data-bearing +cluster. + +## Check Cassandra before migrating + +Confirm the current Cassandra deployment is healthy and record its state: + +```bash +kubectl -n cassandra-system get pods,pvc +kubectl -n cassandra-system exec cassandra-0 -c cassandra -- nodetool status +``` + +All Cassandra pods should be `Running` and report `UN` (Up/Normal) in +`nodetool status`, and the data PersistentVolumeClaims should be `Bound`. +Record the keyspaces and a row count for a known table so you can verify data +after the migration: + +```bash +kubectl -n cassandra-system exec cassandra-0 -c cassandra -- \ + cqlsh -u -p -e "DESCRIBE KEYSPACES;" +``` + +Check the existing schema migration state: + +```sql +SELECT table_name +FROM system_schema.tables +WHERE keyspace_name = 'schema_migrations'; +``` + +For each table returned, run: + +```sql +SELECT version, dirty FROM schema_migrations.; +``` + +Every migration record must have `dirty` set to `false`. If any record is +dirty, reconcile that partial migration before starting this upgrade. + + +Take a backup before migrating. On every node, run +`nodetool flush` and then `nodetool snapshot`, and copy the snapshots off-node. +Do not delete Cassandra PersistentVolumeClaims during any migration method. + + +## Choose a migration method + +Three methods are supported. They are listed in no particular order; choose the +one that fits your deployment and your tolerance for downtime and operational +steps. + +| Method | Downtime | Data handling | Requires | +|---|---|---|---| +| Datacenter expansion | Effectively none | New nodes stream the data; old volumes untouched | Capacity to run both datacenters at once, datacenter-aware clients | +| In-place volume adoption | Brief restart per node | The new pods reuse the existing volumes | A manual StatefulSet recreate; a subPath or a one-time layout relocation | +| Backup and restore | Maintenance window | Snapshot the old cluster, load into a fresh one | Capacity for a parallel cluster during the copy | + +Datacenter expansion keeps the cluster online by adding the new stack as a +second Cassandra datacenter and streaming data to it, then cutting clients over. +It has the most moving parts and needs enough capacity to run both datacenters +during the migration. + +In-place volume adoption reuses the existing data volumes. It is the smallest +change in footprint, but it requires deleting and recreating the Cassandra +StatefulSet (its volumes are retained) and pointing the new pods at the existing +Bitnami on-disk layout. + +Backup and restore is the most conservative for data integrity: the existing +cluster is left untouched until a fresh 0.6.1 cluster is verified. It needs a +maintenance window and capacity to run both clusters during the copy. + +## Method: datacenter expansion + +Add the 0.6.1 stack as a second Cassandra datacenter in the same cluster, +replicate the data to it, move clients over, then remove the old datacenter. +Because the new nodes bootstrap empty and receive their data by streaming, this +method avoids the volume-layout and configuration-compatibility concerns of an +in-place adoption. + + +Validate this procedure in a non-production environment before running it +against production. Perform the keyspace replication and decommission steps +deliberately; an incorrect replication change or an early decommission can lose +data. + + + +Two settings on the new datacenter's nodes decide whether the join and rebuild +succeed: + +- `storage_compatibility_mode` must match the existing cluster. The existing + Cassandra fleet runs `NONE` (full Cassandra 5.0 native format), and the 0.6.1 + chart defaults the new nodes to `NONE` for this reason. If the new nodes run a + different mode (the Apache Cassandra 5.0 default is `CASSANDRA_4`), gossip and + schema exchange still succeed and the datacenters appear healthy, but the + `nodetool rebuild` in step 4 fails: the streaming connection is rejected as + version-incompatible. +- Token allocation. When the existing cluster already holds a large schema, a + node that joins with schema-aware token allocation + (`allocate_tokens_for_local_replication_factor`) must reach schema readiness + within a fixed window during bootstrap, or it aborts with a "Could not achieve + schema readiness" error and restarts. If the new nodes fail to finish joining + with that error, have them allocate tokens without the schema-aware option + (random tokens are acceptable for a new, empty datacenter) and retry. + + +1. Stand up the new datacenter joined to the existing cluster. Deploy the 0.6.1 + Cassandra nodes with the same cluster name as the existing cluster, a + distinct datacenter name, seeds that reach the existing datacenter, and + `auto_bootstrap: false` so the new nodes join without streaming during + bootstrap. Do not run the schema-initialization or migration jobs yet. The + schema already exists in the cluster, and the new nodes receive it through + gossip. Disable both jobs in the new datacenter's values with + `cassandra.hooks.initializeCluster.enabled: false` and + `cassandra.hooks.migrations.enabled: false`. Keep them disabled until the new + datacenter is included in the application and migration-state keyspaces in + step 3. + +2. Confirm both datacenters are visible and healthy from a node in either + datacenter: + + ```bash + kubectl -n cassandra-system exec cassandra-0 -c cassandra -- nodetool status + ``` + + Every node should report `UN`. The new datacenter nodes appear with no or + minimal load until the rebuild in step 4. + +3. Extend replication for every application keyspace, `schema_migrations`, + `system_auth`, `system_distributed`, and `system_traces` to include the new + datacenter. Replicating `schema_migrations` preserves the version and dirty + state that the 0.6.1 migration job uses to apply only pending migrations. + For each keyspace: + + ```sql + ALTER KEYSPACE WITH replication = { + 'class': 'NetworkTopologyStrategy', + '': , + '': + }; + ``` + +4. On each node in the new datacenter, stream the existing data across: + + ```bash + kubectl -n cassandra-system exec -c cassandra -- \ + nodetool rebuild -- + ``` + +5. Run the pending 0.6.1 schema migrations before moving clients. Keep cluster + initialization disabled and enable only the migration hook: + + ```yaml + cassandra: + hooks: + initializeCluster: + enabled: false + migrations: + enabled: true + ``` + + Apply the Cassandra release and wait for its migration job: + + ```bash + set -euo pipefail + make install HELMFILE_ENV="$HELMFILE_ENV" HELMFILE_SELECTOR=name=cassandra + kubectl -n cassandra-system wait \ + --for=condition=complete job/cassandra-migrations --timeout=10m + kubectl -n cassandra-system logs job/cassandra-migrations + ``` + + The logs must end with `All Cassandra migrations completed successfully`. + Do not continue if the job failed or timed out. + +6. Inventory application keyspace replication after the migration job: + + ```sql + SELECT keyspace_name, replication + FROM system_schema.keyspaces; + ``` + + For each application keyspace that does not list both datacenters, update its + replication: + + ```sql + ALTER KEYSPACE WITH replication = { + 'class': 'NetworkTopologyStrategy', + '': , + '': + }; + ``` + + Run the `system_schema.keyspaces` query again. Confirm that every application + keyspace lists both datacenters before continuing. + + If a newly created keyspace already contains data, rebuild that keyspace on + each new-datacenter node before continuing: + + ```bash + kubectl -n cassandra-system exec -c cassandra -- \ + nodetool rebuild -ks -- + ``` + +7. Confirm the migration state is clean: + + ```sql + SELECT table_name + FROM system_schema.tables + WHERE keyspace_name = 'schema_migrations'; + + SELECT version, dirty FROM schema_migrations.; + ``` + + Run the second query for each table returned by the first query. Each query + must return one record with `dirty` set to `false`. This confirms that the + cluster can receive later schema migrations. Do not move clients or remove + the old datacenter while a migration is missing or dirty. + +8. Move clients to the new datacenter. Point application traffic at the new + datacenter using `LOCAL_QUORUM` against the new datacenter name, and confirm + reads and writes succeed there. + +9. Remove the old datacenter from replication, then decommission it. Alter every + keyspace again to drop the old datacenter from the replication map, then + decommission the old-datacenter nodes and remove the old stack. + +## Method: in-place volume adoption + +Reuse the existing data volumes with the new stack. The StatefulSet must be +recreated because its shape is immutable, and the new pods must be pointed at +the Bitnami on-disk layout. + + +The StatefulSet recreate step deletes the StatefulSet object but keeps its pods +and PersistentVolumeClaims. Do not delete the PersistentVolumeClaims. Confirm +the StatefulSet's `persistentVolumeClaimRetentionPolicy.whenDeleted` is not +`Delete` before deleting it. + + +The chart supports adopting the existing volume with `persistence.subPath: data`, +which surfaces the Bitnami nested layout at the paths the official image expects. +A one-time relocation script is also available to convert the volume to the +official layout so no subPath is needed afterward; use one or the other. + + +This method is validated for the standard NVCF Cassandra configuration. It also +depends on the new nodes reading the data the old fleet wrote, which requires the +0.6.1 `cassandra.yaml` to match the settings the old data was written with. The +chart carries the settings the default NVCF deployment needs. If your Bitnami +deployment ran a customized `cassandra.yaml`, verify those settings carry over to +0.6.1 before you commit to this method: a missed setting can prevent the node +from starting on the existing data. When in doubt, use datacenter expansion or +backup and restore, which do not depend on on-disk configuration compatibility. + + +1. In `environments/.yaml`, add `persistence.subPath` under the + existing `cassandra` section. Do not replace the rest of the file: + + ```yaml + cassandra: + persistence: + subPath: "data" + ``` + + This is the only migration-specific override required for the standard NVCF + Cassandra configuration. The chart already defaults + `podSecurityContext.fsGroup` to `999`, `cluster.name` to `cassandra`, and the + image to the 0.6.1 runtime. If the existing deployment customized its cluster + name or Cassandra configuration, carry those matching values forward. + +2. Recreate the Cassandra StatefulSet so the new-shape StatefulSet is created and + re-adopts the existing volumes: + + ```bash + kubectl -n cassandra-system delete statefulset cassandra --cascade=orphan + kubectl -n cassandra-system delete pod cassandra-0 + make install HELMFILE_ENV="$HELMFILE_ENV" HELMFILE_SELECTOR=name=cassandra + ``` + + For a multi-node cluster, the StatefulSet controller recreates `cassandra-0` + and then automatically replaces the remaining pods in reverse ordinal order, + one at a time. Do not delete the remaining pods manually. In another + terminal, monitor the automatic rollout: + + ```bash + kubectl -n cassandra-system rollout status statefulset/cassandra \ + --timeout=30m + ``` + + If the rollout stalls, inspect the pod status, events, and Cassandra ring + before taking manual action. + +3. Confirm the new pod comes up on the existing data: + + ```bash + kubectl -n cassandra-system exec cassandra-0 -c cassandra -- nodetool status + ``` + + The node should report `UN`, and the row count for the known table you + recorded earlier should match. + +## Method: backup and restore + +Leave the existing cluster in place and load its data into a fresh 0.6.1 +cluster. + +1. Snapshot the existing cluster: on every node, run `nodetool flush` then + `nodetool snapshot`, and copy the snapshots off-node. + +2. Deploy a fresh 0.6.1 Cassandra cluster on new volumes and let the init and + migration jobs create the schema. + +3. Restore the data into the new cluster with `sstableloader`, or place the + snapshot sstables and run `nodetool refresh` per table. + +4. Verify row counts and application connectivity against the new cluster, then + decommission the old cluster. + +## Verify the migration + +After any method, confirm the Cassandra cluster is healthy on the 0.6.1 image: + +```bash +kubectl -n cassandra-system get pods +kubectl -n cassandra-system exec cassandra-0 -c cassandra -- nodetool status +``` + +All Cassandra pods should be `Running` and report `UN`. Confirm the known table +row count matches what you recorded before the migration, and confirm the +control plane is healthy: + +```bash +kubectl get pods -A --no-headers | awk \ + '$4 != "Running" && $4 != "Completed" { print } + $4 == "Running" { + split($3, ready, "/") + if (ready[1] != ready[2]) print + }' +``` + +This command should print nothing. It prints pods that are not `Running` or +`Completed`, and `Running` pods whose ready-container count does not match their +total-container count. + +Finally, query each table in the `schema_migrations` keyspace as described in +the pre-migration check. Every record must have `dirty` set to `false` before +you consider the upgrade complete. diff --git a/docs/overview/release-notes/0.6.0.md b/docs/overview/release-notes/0.6.0.md new file mode 100644 index 0000000000..f2ab855d4c --- /dev/null +++ b/docs/overview/release-notes/0.6.0.md @@ -0,0 +1,115 @@ +# NVCF Self-Hosted 0.6.0 Release Notes + +NVCF Self-Hosted 0.6.0 is a stable release. It adds asynchronous task +workloads, multi-cluster control-plane management, OpenAI-compatible LLM +routing, single-mode Multi-Instance GPU (MIG) support, and control-plane +observability. It also adds Early Access support for optional NVIDIA Dynamo and +Grove operator integration. + +## Highlights + +| Feature | Status | Summary | +| --- | --- | --- | +| NVIDIA Cloud Tasks (NVCT) | GA | Run asynchronous GPU jobs to completion without keeping an online endpoint. | +| Multi-cluster GPU management | GA | Register and manage multiple GPU clusters from one NVCF control plane. | +| Dynamo operator integration | Early Access, optional | Install optional NVIDIA Dynamo and Grove operators on GPU clusters. | +| LLM API Gateway | GA | Route OpenAI-compatible chat completions, Responses, and embeddings by model. | +| Single-mode MIG | GA | Expose single-strategy MIG profiles as NVCF instance types. | +| Control-plane observability | GA | Export metrics, logs, and optional traces from control-plane services. | + +## NVIDIA Cloud Tasks (NVCT) + +NVIDIA Cloud Tasks (NVCT) adds an asynchronous workload type for +run-to-completion GPU jobs. Tasks are the batch counterpart to Cloud Functions: +functions stay online to serve invocation requests, while tasks run to +completion and release GPU capacity when they finish. + +Use NVCT for batch inference, fine-tuning, evaluation, solvers, data +preparation, and other GPU workloads that do not require an always-on endpoint. + +## Multi-Cluster GPU Management + +A single NVCF control plane can now register and manage multiple GPU clusters. +Each cluster keeps its own identity and configuration, so operators can expand +across regions, cloud providers, or on-premises environments without running a +separate NVCF control plane for each cluster. + +After registration, workloads can deploy to the available clusters through the +same control-plane API. + +## Dynamo Operator Integration + +NVCF can integrate with optional NVIDIA Dynamo and Grove operators installed on +each GPU cluster. KAI Scheduler is a separate prerequisite for using these +components. + +This integration is Early Access and optional. + +## LLM API Gateway + +The LLM API Gateway provides an OpenAI-compatible entry point for LLM workloads +on NVCF. It reads the target function and model from the request's `model` +field, routes chat completions, Responses, and embeddings to the matching model +backend, and applies per-model token rate limits. + +When cache-aware routing is enabled, the gateway can keep multi-turn requests on +the same backend for better cache locality. + +## Single-Mode MIG + +NVCF can detect GPUs configured with the single MIG strategy and expose each MIG +profile as an NVCF instance type. Workloads can target an isolated GPU slice +instead of a full GPU, improving capacity use on supported nodes. + +## Control-Plane Observability + +Control-plane observability is configured through one Helmfile configuration +block. NVCF can export Prometheus-compatible metrics, control-plane logs, and +optional OpenTelemetry Protocol (OTLP) traces to an existing observability +backend, including in air-gapped environments. + +## Upgrade Notes + +For an existing 0.5.x deployment, follow the +[0.5.0 to 0.6.0 Upgrade](./0.5.0-to-0.6.0-upgrade.md) procedure. It covers the +required migration stops, full stack sync, and GPU cluster re-registration +steps. + +Customers with GPU clusters running NVIDIA Cluster Agent (NVCA) versions older +than 2.51.0 must upgrade those clusters to NVCA 2.51.0 before moving to NVCA +3.x. NVCA 2.51.0 includes fixes for workload lifecycle management and GPU +capacity accounting. Skipping this version can cause incorrect GPU resource +accounting, untracked workload pods, or capacity mismatches between the GPU +cluster and the control plane. + +| Current NVCA version | Action | +| --- | --- | +| 2.50.5 or older | Upgrade to 2.51.0 first, verify cluster health, then upgrade to 3.0 or later. | +| 2.51.0 | Upgrade directly to 3.0 or later. | + +Check the current NVCA version: + +```bash +kubectl get nvcfbackend -n nvca-operator +``` + +The `VERSION` column shows the deployed NVCA version. + +Upgrade the NVCA operator to 2.51.0 before continuing to 3.x: + +```bash +helm upgrade nvca-operator -n nvca-operator \ + --reuse-values \ + --wait \ + "${CHART_URL}" \ + --username='$oauthtoken' \ + --password="$(helm get values -n nvca-operator nvca-operator -o json | jq -r '.ngcConfig.serviceKey')" \ + --set helmManaged.nvcaVersion="2.51.0" +``` + +After the upgrade, verify that the cluster is healthy before proceeding: + +```bash +kubectl get nvcfbackend -n nvca-operator +# Verify VERSION shows 2.51.0 and HEALTH shows healthy. +``` diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/release-notes/0.6.1-to-0.20.6-upgrade.md b/docs/overview/release-notes/0.6.1-to-1.0-upgrade.md similarity index 62% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/release-notes/0.6.1-to-0.20.6-upgrade.md rename to docs/overview/release-notes/0.6.1-to-1.0-upgrade.md index af5e16301a..6d3f715f94 100644 --- a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/release-notes/0.6.1-to-0.20.6-upgrade.md +++ b/docs/overview/release-notes/0.6.1-to-1.0-upgrade.md @@ -1,11 +1,11 @@ -# Upgrade from NVCF Self-Managed 0.6.1 to 0.20.6 +# Upgrade from NVCF Self-Managed 0.6.1 to 1.0 This page will document the validated upgrade from -`nvcf-self-managed-stack` 0.6.1 to the following qualified stack set: +`nvcf-self-managed-stack` 0.6.1 to the 1.0 stack releases: -- Control plane: `nvcf-self-managed-stack` 0.20.6 -- Compute plane: `nvcf-compute-plane-stack` 0.4.4 -- Observability: `nvcf-observability-stack` 0.2.2 +- Self-managed (control plane): `nvcf-self-managed-stack` 1.0.0 +- Compute plane: `nvcf-compute-plane-stack` 1.0.0 +- Observability: `nvcf-observability-stack` 1.0.0 The upgrade procedure is still being validated. Do not use this placeholder as diff --git a/docs/overview/release-notes/0.6.1.md b/docs/overview/release-notes/0.6.1.md new file mode 100644 index 0000000000..ea6105dc66 --- /dev/null +++ b/docs/overview/release-notes/0.6.1.md @@ -0,0 +1,312 @@ +# NVCF Self-Hosted 0.6.1 Release Notes + +NVCF Self-Hosted 0.6.1 is a maintenance release. It updates OpenBao to address +security issues in the runtime and migration images. It also replaces the +archived Bitnami Cassandra runtime with the official Apache Cassandra 5.0.8 +image and an in-house Cassandra Helm chart. + +## Highlights + +| Change | Status | Summary | +| --- | --- | --- | +| OpenBao security update | GA | OpenBao uses the remediated runtime and migration images. Existing installations require a controlled server pod rotation after the Helm upgrade. | +| Cassandra off Bitnami | GA | Cassandra runs on the official Apache Cassandra 5.0.8 image, deployed by an in-house Helm chart, replacing the Bitnami image and subchart. | + +## Cassandra runtime and chart + +The Cassandra runtime image is now built on the official Apache Cassandra 5.0.8 +image instead of the archived Bitnami image, and the Bitnami Cassandra Helm +subchart is replaced by an in-house StatefulSet chart. This removes the +CVE-affected Bitnami base and its bundled components while keeping the same +operational surface: PasswordAuthenticator and CassandraAuthorizer, stable-DNS +seed discovery, the Prometheus metrics exporter, keyspace initialization, and +schema migrations. + +The chart runs the nodes with `storage_compatibility_mode: NONE` to match the +existing fleet's on-disk and streaming format. + +## Upgrade Notes + +Fresh installs need no special handling; Cassandra comes up on the official +image with the in-house chart. + +Existing clusters that already hold data need a migration, because the runtime +image, the Helm chart shape, and the on-disk data layout all change. A normal +in-place Helm upgrade does not carry an existing Bitnami-based cluster onto the +new stack. Follow the +[0.6.0 to 0.6.1 Upgrade](./0.6.0-to-0.6.1-upgrade.md) procedure, which describes +the supported migration methods (datacenter expansion, in-place volume +adoption, backup and restore) and their trade-offs. + +## OpenBao security update + +The OpenBao runtime and migration images include security fixes. OpenBao stores +control-plane secrets and uses persistent Raft storage, so upgrade it before +applying the rest of the patch stack. + +The OpenBao StatefulSet uses `OnDelete`. The Helm upgrade updates the +StatefulSet template and runs the post-upgrade migration Job, but Kubernetes +does not restart existing OpenBao server pods. After the OpenBao release is +synced from the 0.6.1 stack, rotate the OpenBao server pods manually. + +The pod rotation steps recreate pods from the existing StatefulSet. They do not +delete or recreate the OpenBao persistent volume claims (PVCs). + +Troubleshooting OpenBao itself is outside the scope of this upgrade guide. If +OpenBao is sealed, uninitialized, unhealthy, or failing migration before or +during the procedure, stop and resolve OpenBao before continuing. + + +These steps assume 0.6.0 is already deployed and healthy, that you have +downloaded the 0.6.1 stack, and that you run `make` from the extracted +`nvcf-self-managed-stack/` directory with your environment set. + + +```bash +export HELMFILE_ENV= +``` + +### Prepare the 0.6.1 configuration + +Return to the extracted 0.6.1 stack before running any 0.6.1 sync command. +Reconcile the site-specific settings from the 0.6.0 deployment into the 0.6.1 +configuration files: + +- `environments/$HELMFILE_ENV.yaml` +- `secrets/$HELMFILE_ENV-secrets.yaml` + +Start from the files or templates included with 0.6.1 and carry forward the +required values from 0.6.0. Do not replace the 0.6.1 files wholesale because +available settings can change between releases. Preserve the existing registry, +storage, endpoint, credential, and secret values unless this procedure +explicitly instructs you to change them. + +Confirm that both files exist before continuing: + +```bash +for file in \ + "environments/${HELMFILE_ENV}.yaml" \ + "secrets/${HELMFILE_ENV}-secrets.yaml"; do + [ -f "$file" ] || { echo "missing required file: $file"; exit 1; } +done +``` + +### Check OpenBao before the patch + +Confirm that the current OpenBao release is healthy before starting the patch +upgrade: + +```bash +kubectl -n vault-system get pods,jobs +kubectl -n vault-system get statefulset openbao-server \ + -o jsonpath='{range .spec.template.spec.containers[*]}{.name}{": "}{.image}{"\n"}{end}{"updateStrategy: "}{.spec.updateStrategy.type}{"\n"}' +``` + +The OpenBao server pods should be `Running`, the OpenBao Jobs should be +`Complete`, and the update strategy should be `OnDelete`. Record the full image +references printed from the StatefulSet template. These are OpenBao image +references, not NVCF stack version numbers. After step 1, the StatefulSet +template should show the updated OpenBao image reference. Existing pods can +still show the previous OpenBao image reference until step 2 is complete. + +Inspect each OpenBao server pod and record which pod is active: + +```bash +for pod in $(kubectl -n vault-system get pods \ + -l app.kubernetes.io/name=openbao,component=server \ + -o jsonpath='{.items[*].metadata.name}'); do + echo "== $pod ==" + kubectl -n vault-system exec "$pod" -c openbao -- sh -c \ + 'BAO_ADDR=http://127.0.0.1:8200 bao status' +done +``` + +In the command output, the active pod reports `HA Mode active`. Standby pods +report `HA Mode standby` and include the active node address. + +Do not continue if any pod is sealed, uninitialized, or not ready. + +### Step 1: sync OpenBao from the 0.6.1 stack + +Sync only the OpenBao release first. This updates the OpenBao release and +StatefulSet template, rolls the injector deployment, and runs the OpenBao +post-upgrade migration Job. + +```bash +make install HELMFILE_ENV="$HELMFILE_ENV" HELMFILE_SELECTOR=name=openbao-server +``` + +The command blocks until Helm finishes. If the command fails, stop and resolve +the OpenBao release before continuing. Do not rotate OpenBao server pods until +the post-upgrade migration Job has completed. + +Confirm the OpenBao migration Job succeeded: + +```bash +kubectl -n vault-system get jobs +# the openbao-server-migrations Job should report COMPLETIONS 1/1 +``` + +If the migration Job fails, stop and resolve the OpenBao release before +continuing. Do not rotate OpenBao server pods while the migration Job is failed +or incomplete. + +Confirm that the StatefulSet template now points at the updated OpenBao image +reference and still uses `OnDelete`: + +```bash +kubectl -n vault-system get statefulset openbao-server \ + -o jsonpath='{range .spec.template.spec.containers[*]}{.name}{": "}{.image}{"\n"}{end}{"updateStrategy: "}{.spec.updateStrategy.type}{"\n"}' +``` + +Confirm that the injector now injects the updated OpenBao image reference: + +```bash +kubectl -n vault-system get deployment openbao-server-agent-injector \ + -o jsonpath='{range .spec.template.spec.containers[*].env[?(@.name=="AGENT_INJECT_VAULT_IMAGE")]}{.value}{"\n"}{end}' +``` + + +At this point the OpenBao Helm release and StatefulSet template are updated, but +existing OpenBao server pods can still be running the previous OpenBao image. +This is expected because the StatefulSet uses `OnDelete`. + + +### Step 2: rotate OpenBao server pods + +Rotate OpenBao server pods manually so each pod is recreated from the updated +StatefulSet template. Rotate standby pods first. Rotate the active pod last. + + +Do not delete all OpenBao server pods at once. Delete one pod, wait for it to +become ready, and verify that it is initialized and unsealed before deleting the +next pod. Do not delete the OpenBao StatefulSet or its PVCs. + + + +If your installation has only one OpenBao server pod, there are no standby pods. +Rotate the single pod after the migration Job completes, then wait for it to +become ready and verify that it is initialized and unsealed. + + +List the pods and identify the standby pods: + +```bash +for pod in $(kubectl -n vault-system get pods \ + -l app.kubernetes.io/name=openbao,component=server \ + -o jsonpath='{.items[*].metadata.name}'); do + echo "== $pod ==" + kubectl -n vault-system exec "$pod" -c openbao -- sh -c \ + 'BAO_ADDR=http://127.0.0.1:8200 bao status' +done +``` + +In the command output, rotate pods that report `HA Mode standby` first. Do not +rotate the pod that reports `HA Mode active` until all standby pods have been +recreated and verified. + +For each standby pod, delete the pod and wait for it to return. Repeat this +block for every standby pod: + +```bash +export POD= + +# Waits for the old pod to terminate, not for the replacement pod to be ready. +kubectl -n vault-system delete pod "$POD" --wait=true +kubectl -n vault-system wait "pod/$POD" --for=condition=Ready --timeout=10m +kubectl -n vault-system exec "$POD" -c openbao -- sh -c \ + 'BAO_ADDR=http://127.0.0.1:8200 bao status' +``` + +The recreated pod must report `Initialized true`, `Sealed false`, and a healthy +HA mode before you rotate the next pod. + +After all standby pods are recreated and healthy, identify the active pod again: + +```bash +for pod in $(kubectl -n vault-system get pods \ + -l app.kubernetes.io/name=openbao,component=server \ + -o jsonpath='{.items[*].metadata.name}'); do + echo "== $pod ==" + kubectl -n vault-system exec "$pod" -c openbao -- sh -c \ + 'BAO_ADDR=http://127.0.0.1:8200 bao status' +done +``` + +The active pod is the pod that reports `HA Mode active`. + +Rotate the active pod last: + +```bash +export POD= + +# Waits for the old pod to terminate, not for the replacement pod to be ready. +kubectl -n vault-system delete pod "$POD" --wait=true +kubectl -n vault-system wait "pod/$POD" --for=condition=Ready --timeout=10m +kubectl -n vault-system exec "$POD" -c openbao -- sh -c \ + 'BAO_ADDR=http://127.0.0.1:8200 bao status' +``` + +If a recreated pod does not become ready before the timeout, or if it reports +`Sealed true` or `Initialized false`, stop the procedure and resolve that pod +before rotating any other OpenBao server pod. + +### Verify OpenBao after rotation + +Confirm that all OpenBao server pods are ready and use the updated OpenBao image +reference from the StatefulSet template: + +```bash +kubectl -n vault-system get pods \ + -l app.kubernetes.io/name=openbao,component=server + +kubectl -n vault-system get pods \ + -l app.kubernetes.io/name=openbao,component=server \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{range .spec.containers[*]}{" "}{.name}{": "}{.image}{"\n"}{end}{end}' +``` + +Confirm that the StatefulSet reports all replicas updated and ready: + +```bash +kubectl -n vault-system get statefulset openbao-server \ + -o jsonpath='replicas: {.status.replicas}{"\n"}readyReplicas: {.status.readyReplicas}{"\n"}updatedReplicas: {.status.updatedReplicas}{"\n"}' +``` + +The `replicas`, `readyReplicas`, and `updatedReplicas` values should match. +If `updatedReplicas` briefly lags after the final pod is recreated, wait a few +seconds and rerun the command. The per-pod image reference check above and +`bao status` check below are the primary verification points. + +Confirm that each OpenBao server pod is initialized, unsealed, and participating +in HA: + +```bash +for pod in $(kubectl -n vault-system get pods \ + -l app.kubernetes.io/name=openbao,component=server \ + -o jsonpath='{.items[*].metadata.name}'); do + echo "== $pod ==" + kubectl -n vault-system exec "$pod" -c openbao -- sh -c \ + 'BAO_ADDR=http://127.0.0.1:8200 bao status' +done +``` + +Do not continue if any pod is sealed, uninitialized, not ready, or still running +the previous OpenBao image. + +### Sync the remaining 0.6.1 stack + +After OpenBao is healthy on the updated OpenBao image, sync the rest of the +0.6.1 stack: + +```bash +make install HELMFILE_ENV="$HELMFILE_ENV" +``` + +Confirm that the control plane is healthy: + +```bash +kubectl get pods -A +``` + +This command should print nothing. Any output is a pod that is not yet +`Running` or `Completed`. diff --git a/docs/overview/release-notes/1.0.md b/docs/overview/release-notes/1.0.md new file mode 100644 index 0000000000..94f37b72a6 --- /dev/null +++ b/docs/overview/release-notes/1.0.md @@ -0,0 +1,38 @@ +# NVCF 1.0 Release Notes + +NVCF 1.0 is the first release in which the three Helm stacks are versioned +and released independently. Each stack starts its release history at 1.0.0 +and advances on its own release train from here. See the +[compatibility matrix](../compatibility-matrix.md) for the stack releases that +are qualified to run together. + + +Detailed changes, known issues, and compatibility notes for each stack will be +added before the release is finalized. + + +## Self-Managed Stack 1.0.0 + +Release: `nvcf-self-managed-stack` 1.0.0, tagged from +`deploy/stacks/self-managed/v1.0.0`. This release is the same content as +0.20.6, retagged to start the 1.x line. + +- Changes: to be added. +- Known issues: to be added. +- Upgrade: [Upgrade from 0.6.1 to 1.0](./0.6.1-to-1.0-upgrade.md). + +## Compute Plane Stack 1.0.0 + +Release: `nvcf-compute-plane-stack` 1.0.0, tagged from +`deploy/stacks/nvcf-compute-plane/v1.0.0`. + +- Changes: to be added. +- Known issues: to be added. + +## Observability Stack 1.0.0 + +Release: `nvcf-observability-stack` 1.0.0, tagged from +`deploy/stacks/observability/v1.0.0`. + +- Changes: to be added. +- Known issues: to be added. diff --git a/docs/overview/release-notes/index.md b/docs/overview/release-notes/index.md new file mode 100644 index 0000000000..3724adbf34 --- /dev/null +++ b/docs/overview/release-notes/index.md @@ -0,0 +1,19 @@ +# Release Notes + +Changes, fixes, and upgrade procedures for NVCF releases. From 1.0 onward each +Helm stack releases independently; the [compatibility matrix](../compatibility-matrix.md) +lists the stack releases that are qualified to run together. + +## Current + +- [1.0 Release Notes](./1.0.md) +- [Upgrade from 0.6.1 to 1.0](./0.6.1-to-1.0-upgrade.md) + +## Legacy + +Releases before 1.0 shipped all components as one self-managed release. + +- [0.6.1 Release Notes](./0.6.1.md) +- [0.6.0 to 0.6.1 Upgrade](./0.6.0-to-0.6.1-upgrade.md) +- [0.6.0 Release Notes](./0.6.0.md) +- [0.5.0 to 0.6.0 Upgrade](./0.5.0-to-0.6.0-upgrade.md) diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/samples/configs/cp-env-eks-example.yaml b/docs/overview/samples/configs/cp-env-eks-example.yaml similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/samples/configs/cp-env-eks-example.yaml rename to docs/overview/samples/configs/cp-env-eks-example.yaml diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/samples/configs/cp-example-secrets.yaml b/docs/overview/samples/configs/cp-example-secrets.yaml similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/samples/configs/cp-example-secrets.yaml rename to docs/overview/samples/configs/cp-example-secrets.yaml diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/samples/configs/local-dev-env.yaml b/docs/overview/samples/configs/local-dev-env.yaml similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/samples/configs/local-dev-env.yaml rename to docs/overview/samples/configs/local-dev-env.yaml diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/samples/scripts/create-nvcr-pull-secrets.sh b/docs/overview/samples/scripts/create-nvcr-pull-secrets.sh similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/samples/scripts/create-nvcr-pull-secrets.sh rename to docs/overview/samples/scripts/create-nvcr-pull-secrets.sh diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/samples/scripts/force-cleanup-nvcf.sh b/docs/overview/samples/scripts/force-cleanup-nvcf.sh similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/samples/scripts/force-cleanup-nvcf.sh rename to docs/overview/samples/scripts/force-cleanup-nvcf.sh diff --git a/docs/user/streaming-functions.md b/docs/overview/streaming-functions.md similarity index 98% rename from docs/user/streaming-functions.md rename to docs/overview/streaming-functions.md index 12c0bb1937..c410c33685 100644 --- a/docs/user/streaming-functions.md +++ b/docs/overview/streaming-functions.md @@ -58,13 +58,13 @@ done If the NIC NSG rule is missing, WebRTC clients receive `NVST_R_GENERIC_ERROR` while the function remains ACTIVE. See -[streaming-troubleshooting](./troubleshooting.md#webrtc-streaming-fails-after-function-shows-active) +[streaming-troubleshooting](/nvcf/self-managed/troubleshooting#webrtc-streaming-fails-after-function-shows-active) for diagnosis steps. ### AWS (EKS) For LLS streaming on EKS, security group and NLB configuration is covered -in [LLS Installation](./lls-installation.md#security-group). +in [LLS Installation](/nvcf/self-managed/lls-installation#security-group). ## Building the Streaming Server Application @@ -205,7 +205,7 @@ This example focuses on NVCF integration. In production, you should also impleme For certain applications, TLS/SSL support is required. The proxy can be configured to use self-signed certificates for development and testing purposes by setting `PROXY_SSL_INSECURE=true`. -Update `NVCF_SERVER` to point to your gateway address. See [gateway-routing](./gateway-routing.md) for details. +Update `NVCF_SERVER` to point to your gateway address. See [gateway-routing](/nvcf/self-managed/gateway-routing) for details. diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/task-creation.md b/docs/overview/task-creation.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/task-creation.md rename to docs/overview/task-creation.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/api.md b/docs/self-managed-1.0/api.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/api.md rename to docs/self-managed-1.0/api.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/autoscaling/architecture.md b/docs/self-managed-1.0/autoscaling/architecture.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/autoscaling/architecture.md rename to docs/self-managed-1.0/autoscaling/architecture.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/autoscaling/index.md b/docs/self-managed-1.0/autoscaling/index.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/autoscaling/index.md rename to docs/self-managed-1.0/autoscaling/index.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/autoscaling/observability.md b/docs/self-managed-1.0/autoscaling/observability.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/autoscaling/observability.md rename to docs/self-managed-1.0/autoscaling/observability.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/autoscaling/operations.md b/docs/self-managed-1.0/autoscaling/operations.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/autoscaling/operations.md rename to docs/self-managed-1.0/autoscaling/operations.md diff --git a/docs/self-managed-1.0/caches.md b/docs/self-managed-1.0/caches.md new file mode 100644 index 0000000000..4873277339 --- /dev/null +++ b/docs/self-managed-1.0/caches.md @@ -0,0 +1,47 @@ +# Simulation Cluster Caches + +This section covers cache components for self-hosted NVCF deployments. Caches improve performance by storing frequently accessed content locally, reducing network bandwidth usage and accelerating scene loading. + +## Overview + +Self-hosted NVCF supports several cache components: + +- **Derived Data Cache Service (DDCS)** - Caches derived content to reduce scene load time and improve rendering performance +- **USD Content Cache (UCC)** - Caches USD content from object storage to accelerate scene loading + +## When to Use Caches + +See the individual cache component guides for detailed information on when to use each cache: + +- [Derived Data Cache Service](https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/) - Derived Data Cache Service +- [USD Content Cache](https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/) - USD Content Cache + +## Documentation + +Each cache component has comprehensive documentation covering configuration, deployment, and advanced features: + +## Configuration + +Cache components are configured using Helm values files. Each cache guide includes: + +- Base configuration examples +- Configuration options and parameters +- Performance tuning recommendations +- Best practices + +## Monitoring + +Cache components include Prometheus metrics for monitoring: + +- Cache hit ratios +- Storage utilization +- Request throughput +- Performance metrics + +## Next Steps + +1. **Review cache guides** - Read the detailed guides for caches you want to deploy +2. **Plan your deployment** - Determine which caches fit your use case +3. **Configure caches** - Set up Helm values files based on the examples +4. **Deploy caches** - Install caches using Helm or Helmfile +5. **Monitor performance** - Set up monitoring dashboards and alerts diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cli.md b/docs/self-managed-1.0/cli.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cli.md rename to docs/self-managed-1.0/cli.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/configuration.md b/docs/self-managed-1.0/cluster-management/configuration.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/configuration.md rename to docs/self-managed-1.0/cluster-management/configuration.md diff --git a/docs/self-managed-1.0/cluster-management/container-cache.md b/docs/self-managed-1.0/cluster-management/container-cache.md new file mode 100644 index 0000000000..583026471a --- /dev/null +++ b/docs/self-managed-1.0/cluster-management/container-cache.md @@ -0,0 +1,459 @@ +# Container Cache + +This guide provides detailed information on the Container Cache component and its use in NVCF GPU clusters. + +Container Cache provides a container caching solution specifically optimized for NGC. It is designed to enhance the efficiency of Docker image pulls from NGC by caching the images locally, reducing network bandwidth and improving pull times for frequently accessed images. + +The Container Cache acts as a proxy between your Kubernetes cluster and the NGC registry, caching frequently accessed container images locally to reduce network bandwidth usage and improve deployment times. + +## Installation + +### Prerequisites + +- A running Kubernetes cluster with `kubectl` access +- **Helm** >= 3.12 +- Credentials for the registry where your NVCF charts and images are stored + +### Step 1. Authenticate Helm to your chart registry + +Authenticate Helm to your OCI registry where the NVCF charts are stored: + +```bash +echo "${REGISTRY_PASSWORD}" | helm registry login ${REGISTRY} \ + --username '${REGISTRY_USERNAME}' --password-stdin +``` + +### Step 2. Create the namespace and image pull secret + +```bash +kubectl create namespace container-caching +``` + +Create an image pull secret so that pods can pull container images from your registry. + + + + +```bash +kubectl create secret docker-registry nvcr-creds \ + --docker-server=.dkr.ecr..amazonaws.com \ + --docker-username=AWS \ + --docker-password="$(aws ecr get-login-password --region )" \ + --namespace=container-caching +``` + + + + + +```bash +kubectl create secret docker-registry nvcr-creds \ + --docker-server= \ + --docker-username= \ + --docker-password= \ + --namespace=container-caching +``` + + + + + + +The secret name `nvcr-creds` is referenced in the values file under `images.secrets`. +If you use a different secret name, update the values file to match. + + + +### Step 3. Create a values file + +Create a `values.yaml` using the complete example in [Base Configuration] below. + +- **BYOC users** pulling directly from NGC can use the `nvcr.io/nvidia/nvcf-byoc/` image + paths without mirroring. +- **Self-hosted users** should replace the `/` placeholders with + their mirrored registry path (see the Artifact Manifest in the self-hosted installation guide). + +Adjust `storageClassName` for your environment (e.g., `gp3` for AWS EKS). + +### Step 4. Install the chart + + + + +Replace `/` with your mirrored registry path. + +```bash +helm upgrade --install container-cache \ + oci:////nvcf-container-cache \ + --version 0.25.22 \ + --namespace container-caching \ + --values values.yaml +``` + + + + + +### Step 5. Verify the installation + +Container Cache deploys two workloads: + +- A **StatefulSet** (`container-cache`) with the number of replicas set by `replicaCount`. + Each replica runs two containers (nginx proxy + prometheus exporter) and provisions two PVCs + (`cache` and `proxy-cache`). +- A **DaemonSet** (`container-cache-cc`) that runs on every node and configures the container + runtime (containerd) to route image pulls through the cache. + +```bash +# StatefulSet replicas and DaemonSet pods should all be Running +kubectl get pods -n container-caching + +# Verify the StatefulSet is fully ready (READY should match replicaCount) +kubectl get statefulset -n container-caching + +# Verify the DaemonSet is running on all nodes +kubectl get daemonset -n container-caching + +# Check services are created +kubectl get svc -n container-caching + +# Check persistent volume claims are Bound +kubectl get pvc -n container-caching +``` + +## Base Configuration + +The following is a complete example `values.yaml` for deploying Container Cache. +Copy this file and adjust values for your environment. Each section is explained in detail +below. + +- **BYOC users** can use the `nvcr.io/nvidia/nvcf-byoc/` paths directly. +- **Self-hosted users** should replace `/` with their mirrored + registry path (see the Artifact Manifest in the self-hosted installation guide for source image paths). + +```yaml +replicaCount: 3 + +targetHost: nvcr.io,docker.io + +images: + server: //nvcf-container-cache:v1.1.31 + exporter: nginx/nginx-prometheus-exporter:1.0 + certificates: //nvcf-proxy-tls-certs:1.2.0 + secrets: + - nvcr-creds + +cache: + keyStorageSize: 50m + maxSize: 180g + inactive: 1d + valid: 1h + +persistentVolumeClaim: + sizeGB: 100 + storageClassName: gp3 # Use gp3 for AWS EKS, adjust for other platforms + sizeProxyGB: 100 + +service: + type: ClusterIP + port: 30345 + +metrics: + cacheMetricsStorageSize: 300m + throughputHistogramBuckets: 25000000, 30000000, 35000000, 40000000, 50000000, 60000000, 80000000, 100000000 + +resources: + requests: + memory: 2Gi + cpu: "1" + limits: + memory: 4Gi + cpu: "2" + +traces: + enabled: false + +nucleus: + enabled: false + +vault: + enabled: false + +monitoring: + enabled: false +``` + +## Configuration Sections + +### Replicas + +The number of Container Cache pods is controlled through the `replicaCount` value. Container Cache replicas operate independently and distribute requests from worker nodes. + +```yaml +# values.yaml + +# Min Value: 1 +# Recommended Value: 3 +replicaCount: 3 +``` + + +Container Cache is designed to scale horizontally to handle increased load. + + + +### Node Selection + +Container Cache pods are scheduled on nodes with appropriate labels to ensure they run on compute nodes. Adjust the node selector based on your cluster's node labeling scheme. + +```yaml +# values.yaml + +nodeSelector: + nvcf.nvidia.com/workload: gpu # Adjust based on your node labels, or remove if not using node labels +``` + +### Target Hosts + +The Container Cache can proxy requests to multiple container registries. The default configuration includes both NGC and Docker Hub. + +```yaml +# values.yaml + +# Domain of target Host where the proxy passes the request to +targetHost: nvcr.io,docker.io +``` + +### Image Configuration + +Container Cache uses specific images for the server, exporter, and certificates: + +```yaml +# values.yaml + +images: + # Container Cache Nginx Proxy + server: //nvcf-container-cache:v1.1.31 + + # Nginx Prometheus Exporter (public image, no mirroring required) + exporter: nginx/nginx-prometheus-exporter:1.0 + + # TLS Certificates + certificates: //nvcf-proxy-tls-certs:1.2.0 + + # Image pull secret created in Step 2 + secrets: + - nvcr-creds +``` + +Replace `/` with your registry path. BYOC users pulling +directly from NGC can use `nvcr.io/nvidia/nvcf-byoc` as the registry/repo path. + +### Cache Configuration + +The cache behavior is controlled through several parameters: + +```yaml +# values.yaml + +cache: + # Size for storing cache keys + keyStorageSize: 50m + + # Maximum size of the cache + maxSize: 180g + + # Period a resource can remain in cache without being accessed + inactive: 1d + + # Period a cache is valid if resource doesn't become inactive first + valid: 1h +``` + +### Storage Configuration + +Container Cache requires persistent storage for caching container images: + +```yaml +# values.yaml + +persistentVolumeClaim: + # Size of persistent volume + sizeGB: 100 + + # Storage class for persistent volume claim + storageClassName: gp3 # Use gp3 for Amazon EKS, adjust for other platforms + + # Size of persistent volume for proxy cache + sizeProxyGB: 100 +``` + +### Service Configuration + +The service port is configurable. The service type is not: Container Cache is +always exposed as a `NodePort`. + +```yaml +# values.yaml + +service: + # Port for the Container Cache service + port: 30345 +``` + +The service is always `NodePort` and the type is not configurable. The container +runtime on each node reaches the cache at `${NODE_IP}:${port}` from the host +network namespace, where cluster service DNS and ClusterIP addresses are not +dependable, so the port must be published on every node. + +`service.type` was never a safe setting. A ClusterIP service publishes no node +port, so the registry mirror written to each node points at a port nothing +listens on, and image pulls fall back to the upstream registry with no error and +no cache involvement, which looks identical to a working cache until you check +cache metrics. The chart now fails the render on any value other than `NodePort` +so a leftover override cannot silently disable caching. + +### Metrics Configuration + +Container Cache includes Prometheus metrics for monitoring cache performance: + +```yaml +# values.yaml + +metrics: + # Size for storing cache metrics + cacheMetricsStorageSize: 300m + + # Bucket configuration for throughput histogram + throughputHistogramBuckets: 25000000, 30000000, 35000000, 40000000, 50000000, 60000000, 80000000, 100000000 +``` + +### Resource Requests and Limits + +Resource requests and limits for the Container Cache StatefulSet pods are **required**. The +chart will fail to install without them. + +```yaml +# values.yaml + +resources: + requests: + memory: 2Gi + cpu: "1" + limits: + memory: 4Gi + cpu: "2" +``` + + +Adjust these values based on your cluster size and expected cache throughput. Larger +deployments with high pull rates may need more memory and CPU. + + + +## Architecture + +### Container Cache Architecture + +Container Cache consists of several components: + +1. **Nginx Proxy Server**: Handles incoming requests and serves cached content +2. **Prometheus Exporter**: Provides metrics for monitoring +3. **Persistent Storage**: Stores cached container images +4. **DaemonSet**: Configures containerd on worker nodes to use the cache + +### Data Flow + +1. **Initial Request**: Worker node requests a container image +2. **Cache Check**: Container Cache checks if image is cached locally +3. **Cache Hit**: If cached, serve image directly from local storage +4. **Cache Miss**: If not cached, fetch from upstream registry and cache locally +5. **Response**: Return image to requesting worker node + +## Performance Considerations + +### Cache Size + +The cache size should be sized based on your workload requirements: + +- **Small deployments**: 50-100GB +- **Medium deployments**: 100-500GB +- **Large deployments**: 500GB+ + +### Storage Performance + +For optimal performance, use high-performance storage: + +- **AWS**: Use gp3 or io1/io2 EBS volumes +- **Azure**: Use Premium SSD storage +- **GCP**: Use SSD persistent disks + +### Network Configuration + +Container Cache should be deployed close to worker nodes to minimize network latency: + +- Deploy in the same availability zone as worker nodes +- Use high-bandwidth network connections +- Consider using dedicated network interfaces for cache traffic + +## Monitoring and Observability + +### Metrics + +Container Cache provides several key metrics: + +- **Cache hit ratio**: Percentage of requests served from cache +- **Cache size**: Current size of cached data +- **Request throughput**: Number of requests per second +- **Response times**: Time to serve cached vs. uncached content + +### Logging + +Container Cache logs include: + +- Cache hit/miss events +- Upstream registry communication +- Error conditions and troubleshooting information +- Performance metrics + +## Troubleshooting + +### Common Issues + +**Cache Not Working** +: - Verify containerd configuration on worker nodes + +- Check network connectivity to Container Cache service +- Ensure proper DNS resolution + +**Low Cache Hit Ratio** +: - Review cache size configuration + +- Check cache eviction policies +- Monitor storage performance + +**Storage Issues** +: - Verify storage class availability + +- Check persistent volume claims +- Monitor disk space usage + +## Best Practices + +### Deployment + +- Deploy Container Cache before deploying workloads +- Use multiple replicas for high availability +- Monitor cache performance and adjust configuration as needed + +### Configuration + +- Size cache appropriately for your workload +- Use high-performance storage for better performance +- Configure appropriate cache eviction policies + +### Maintenance + +- Monitor cache hit ratios and adjust cache size as needed +- Regularly review and clean up unused cached images +- Update Container Cache images regularly for security patches diff --git a/docs/self-managed-1.0/cluster-management/gang-scheduling.md b/docs/self-managed-1.0/cluster-management/gang-scheduling.md new file mode 100644 index 0000000000..623a387308 --- /dev/null +++ b/docs/self-managed-1.0/cluster-management/gang-scheduling.md @@ -0,0 +1,160 @@ +# Gang Scheduling + +Gang scheduling holds a group of Pods until the scheduler can place every +required member. This prevents a partial deployment from consuming GPUs while +the remaining Pods stay `Pending`. + +NVCF Helm functions and tasks may use custom resources for several components +for gang-scheduled workloads: + +- [KAI Scheduler](https://github.com/kai-scheduler/KAI-Scheduler) provides + queueing, resource allocation, and atomic placement through `PodGroup` + resources. +- [Grove](https://github.com/NVIDIA/grove) represents related workload roles as + `PodCliqueSet`, `PodClique`, `PodCliqueScalingGroup`, and `PodGang` + resources. Grove delegates placement to KAI Scheduler. +- [Dynamo](https://docs.nvidia.com/dynamo/v1.2.1/kubernetes-deployment/scale/grove) + describes inference services such as frontend, prefill, and decode workers + in a `DynamoGraphDeployment`. The Dynamo operator uses Grove to orchestrate + those services. + +The NVCF Cluster Agent (NVCA) admits the KAI, Grove, and Dynamo resources in +function Helm charts when their compute plane add-ons are enabled. +NVCA also uses KAI implicitly when the `KAIScheduler` feature flag is enabled +to binpack workloads. + +For more background, see the upstream +[KAI gang scheduling guide](https://github.com/kai-scheduler/KAI-Scheduler/blob/main/docs/batch/README.md) +and [Grove core concepts](https://github.com/NVIDIA/grove/blob/main/docs/user-guide/01_core-concepts/01_overview.md). + +## Enable gang scheduling + +Enable KAI Scheduler, Grove or Dynamo when functions use their custom resources: + +```yaml +addons: + kaiScheduler: + enabled: true + groveOperator: + enabled: true + dynamoOperator: + enabled: true +``` + +The compute plane installs the components in dependency order: + +1. KAI Scheduler +2. Grove +3. Dynamo + +The same add-ons configure NVCA: + +- KAI adds the `KAIScheduler` feature gate and permits `PodGroup`. +- Grove permits `PodCliqueSet`, `PodClique`, `PodCliqueScalingGroup`, and + `PodGang`. Grove requires KAI Scheduler to be enabled. +- Dynamo adds the `DynamoOperatorSupport` feature gate and permits Dynamo + custom resources. Dynamo requires Grove to be enabled. + +See [KAI Scheduler](./kai-scheduler.md) for queue configuration and the +standalone installation path. + +## Gang schedule a StatefulSet with KAI + +Use a StatefulSet with parallel Pod management for a direct KAI workload: + +```yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: distributed-worker +spec: + podManagementPolicy: Parallel + replicas: 2 + selector: + matchLabels: + app: distributed-worker + template: + metadata: + labels: + app: distributed-worker + spec: + containers: + - name: worker + image: +``` + +When the `KAIScheduler` feature is enabled, NVCA assigns the KAI scheduler and +queue to the Pod template. KAI creates one `PodGroup` for the StatefulSet and +waits until every replica can be placed. + +Add topology annotations only when the gang must also fit in a specific +hardware domain. See [Topology-Aware Scheduling](./topology-aware-scheduling.md). + + +KAI creates a separate `PodGroup` for each Deployment replica. Use a +StatefulSet when all replicas must be scheduled as one gang. + + +The +[multi-node Helm function sample](https://github.com/NVIDIA/nvcf/tree/main/examples/function-samples/helmchart-samples/multi-node-helm-function-test) +contains optional KAI topology annotations for gangs that also need GPU clique +placement. + +## Gang schedule a Dynamo workload with Grove + +A `DynamoGraphDeployment` groups inference roles in one resource. A multinode +service tells Dynamo and Grove that each replica needs multiple Pods: + +```yaml +apiVersion: nvidia.com/v1alpha1 +kind: DynamoGraphDeployment +metadata: + name: myllm +spec: + services: + VllmDecodeWorker: + componentType: worker + replicas: 1 + multinode: + nodeCount: 2 + resources: + limits: + gpu: "4" +``` + +The Dynamo operator converts the service into Grove resources. Grove groups the +Pods, and KAI reserves and places the complete group. Function authors do not explicitly +create `PodGang` or `PodGroup` resources for a `DynamoGraphDeployment`. + +See the [Dynamo Operator sample](https://github.com/NVIDIA/nvcf/tree/main/examples/function-samples/helmchart-samples/dynamo-operator-sample) +for a disaggregated frontend, prefill, and decode workload. See the upstream +[Dynamo multinode guide](https://docs.nvidia.com/dynamo/v1.2.1/kubernetes-deployment/scale/multinode-deployments) +and [Grove quickstart](https://github.com/NVIDIA/grove/blob/main/docs/quickstart.md) +for advanced workload configuration. + +## Verify gang scheduling (Cluster admins only) + +Check that the operators are running: + +```bash +kubectl get pods -n kai-scheduler +kubectl get pods -n grove-system +kubectl get pods -n dynamo-system +``` + +Inspect the scheduling resources for a deployed function: + +```bash +kubectl get podgroups.scheduling.run.ai -A +kubectl get podgangs.scheduler.grove.io -A +kubectl get podcliquesets.grove.io -A +``` + +If a gang remains `Pending`, describe its Pods and KAI `PodGroup`. Common +causes are insufficient free GPUs, queue limits, or topology constraints that +no available domain can satisfy. + +```bash +kubectl describe pod -n +kubectl describe podgroup -n +``` diff --git a/docs/self-managed-1.0/cluster-management/gxcache.md b/docs/self-managed-1.0/cluster-management/gxcache.md new file mode 100644 index 0000000000..86dbf73650 --- /dev/null +++ b/docs/self-managed-1.0/cluster-management/gxcache.md @@ -0,0 +1,419 @@ +# Distributed Shader Cache (GXCache) + +This guide provides detailed information on the Distributed Shader Cache (GXCache) component and its use in NVCF GPU clusters. + +GXCache is a cloud-native shader cache for NVIDIA that provides distributed shader caching capabilities. It consists of the GXCache server and mutating webhook to optimize shader compilation and caching across your cluster. + +GXCache improves rendering performance by caching compiled shaders, reducing compilation time for frequently used shaders and enabling faster scene loading and rendering. + +## Installation + +### Prerequisites + +- A running Kubernetes cluster with `kubectl` access +- **Helm** >= 3.12 +- Credentials for the registry where your NVCF charts and images are stored + +### Step 1. Authenticate Helm to your chart registry + +Authenticate Helm to your OCI registry where the NVCF charts are stored: + +```bash +echo "${REGISTRY_PASSWORD}" | helm registry login ${REGISTRY} \ + --username '${REGISTRY_USERNAME}' --password-stdin +``` + +### Step 2. Create the namespace and image pull secret + +```bash +kubectl create namespace gxcache +``` + +Create an image pull secret so that pods can pull container images from your registry. + + + + +```bash +kubectl create secret docker-registry nvcr-creds \ + --docker-server=.dkr.ecr..amazonaws.com \ + --docker-username=AWS \ + --docker-password="$(aws ecr get-login-password --region )" \ + --namespace=gxcache +``` + + + + + +```bash +kubectl create secret docker-registry nvcr-creds \ + --docker-server= \ + --docker-username= \ + --docker-password= \ + --namespace=gxcache +``` + + + + + + +The secret name `nvcr-creds` is referenced in the values file under +`webhook.deployment.secret` and `service.deployment.imageSecret`. +If you use a different secret name, update the values file to match. + + + +### Step 3. Create a values file + +Create a `values.yaml` using the complete example in [Base Configuration] below. + +- **BYOC users** pulling directly from NGC can use the `nvcr.io/nvidia/nvcf-byoc/` image + paths without mirroring. +- **Self-hosted users** should replace the `/` placeholders with + their mirrored registry path (see the Artifact Manifest in the self-hosted installation guide). + +At minimum, configure the storage class for your environment (e.g., `gp3` for AWS EKS) +and the node selector for your GPU nodes. + +### Step 4. Install the chart + + + + +Replace `/` with your mirrored registry path. + +```bash +helm upgrade --install gxcache \ + oci:////gxcache \ + --version 0.8.2 \ + --namespace gxcache \ + --values values.yaml +``` + + + + + +### Step 5. Verify the installation + +```bash +# All pods should reach Running status +kubectl get pods -n gxcache + +# Check the service and webhook are available +kubectl get svc -n gxcache + +# Check persistent volume claims are bound +kubectl get pvc -n gxcache +``` + +## Base Configuration + +The following is a complete example `values.yaml` for deploying GXCache. Copy this file and +adjust values for your environment. Each section is explained in detail below. + +- **BYOC users** can use the `nvcr.io/nvidia/nvcf-byoc/` paths directly. +- **Self-hosted users** should replace `/` with their mirrored + registry path (see the Artifact Manifest in the self-hosted installation guide for source image paths). + +```yaml +webhook: + nodeSelector: + node-type: compute # Adjust based on your node labels, or set to null + deployment: + image: //gxcache-webhook + version: 59bd8ec5 + secret: nvcr-creds # Image pull secret created in Step 2 + client: + version: + image: //gxcache-init + tag: 1e47f722 + config: + tls: + enabled: false + metrics: + enabled: false # Requires monitoring.coreos.com/v1 CRD + +service: + nodeSelector: + node-type: compute # Adjust based on your node labels, or set to null + deployment: + image: //gxcache-service:b206ce39 # Tag is part of the image field + imageSecret: nvcr-creds # Image pull secret created in Step 2 + vault: + enabled: false + kns: + enabled: true + keyset: + api: /.well-known/jwks.json + endpoint: http://notary.nvcf.svc.cluster.local:8080 + persistence: + enabled: true + storageClass: gp3 # Use gp3 for AWS EKS, adjust for other platforms + accessMode: ReadWriteOnce + size: 20Gi + metrics: + enabled: false # Requires monitoring.coreos.com/v1 CRD +``` + +Replace `/` with your registry path. BYOC users pulling +directly from NGC can use `nvcr.io/nvidia/nvcf-byoc` as the registry/repo path. + + +If your cluster does not use node labels, set `nodeSelector` to `null`. + + + +## Configuration Sections + +### Webhook Configuration + +The GXCache webhook is responsible for intercepting and modifying pod specifications to enable shader caching: + +```yaml +# values.yaml + +webhook: + nodeSelector: + node-type: compute # Adjust based on your node labels + deployment: + image: //gxcache-webhook + version: 59bd8ec5 + secret: nvcr-creds # Registry credentials secret + client: + version: + image: //gxcache-init + tag: 1e47f722 + config: + tls: + enabled: false + metrics: + enabled: false # Requires monitoring.coreos.com/v1 CRD +``` + +### Service Configuration + +The GXCache service provides the shader cache functionality: + +```yaml +# values.yaml + +service: + nodeSelector: + node-type: compute # Adjust based on your node labels + deployment: + image: //gxcache-service:b206ce39 + imageSecret: nvcr-creds # Registry credentials secret + vault: + enabled: false +``` + +### Node Selection + +GXCache components are scheduled on nodes with appropriate labels to ensure they run on GPU compute nodes. Adjust the node selector based on your cluster's node labeling scheme. + +```yaml +# values.yaml + +webhook: + nodeSelector: + node-type: compute +service: + nodeSelector: + node-type: compute +``` + +### Key Management + +GXCache uses NVIDIA's Key Notary Service (KNS) for secure key management. +For self-hosted deployments use the in-cluster notary service: + +```yaml +# values.yaml + +service: + kns: + enabled: true + keyset: + api: /.well-known/jwks.json + endpoint: http://notary.nvcf.svc.cluster.local:8080 +``` + +### Storage Configuration + +GXCache requires persistent storage for caching shader data: + +```yaml +# values.yaml + +service: + persistence: + enabled: true + storageClass: gp3 # Use gp3 for AWS EKS, adjust for other platforms + accessMode: ReadWriteOnce + size: 20Gi +``` + +### Metrics Configuration + +GXCache supports Prometheus metrics for monitoring: + +```yaml +# values.yaml + +webhook: + metrics: + enabled: false # Requires monitoring.coreos.com/v1 CRD +service: + metrics: + enabled: false # Requires monitoring.coreos.com/v1 CRD +``` + +## Architecture + +### GXCache Architecture + +GXCache consists of two main components: + +1. **GXCache Service**: Provides the shader cache storage and retrieval functionality +2. **GXCache Webhook**: Intercepts pod creation to inject shader cache configuration + +### Data Flow + +1. **Pod Creation**: Application pod is created with GPU requirements +2. **Webhook Interception**: GXCache webhook intercepts pod creation +3. **Configuration Injection**: Webhook injects shader cache configuration +4. **Shader Compilation**: Application compiles shaders during runtime +5. **Cache Storage**: Compiled shaders are stored in GXCache +6. **Cache Retrieval**: Subsequent requests retrieve cached shaders + +## Performance Considerations + +### Storage Performance + +For optimal shader cache performance: + +- **Storage Type**: Use high-performance SSD storage (gp3, io1, io2) +- **Storage Size**: Size based on expected shader cache usage (20GB minimum) +- **Access Mode**: ReadWriteOnce is sufficient for most deployments + +### Network Configuration + +GXCache should be deployed to minimize network latency: + +- Deploy in the same availability zone as GPU nodes +- Use high-bandwidth network connections +- Consider dedicated network interfaces for cache traffic + +### Cache Size Planning + +Plan cache size based on your workload: + +- **Small workloads**: 20-50GB +- **Medium workloads**: 50-200GB +- **Large workloads**: 200GB+ + +## Monitoring and Observability + +### Metrics + +GXCache provides several key metrics: + +- **Cache hit ratio**: Percentage of shader requests served from cache +- **Cache size**: Current size of cached shader data +- **Request latency**: Time to serve cached shaders +- **Storage utilization**: Disk space usage + +### Prometheus Integration + +To enable Prometheus metrics: + +1. **Install Prometheus Operator** (if not already installed): + + ```bash + helm install --wait --timeout 15m \ + --namespace monitoring --create-namespace \ + --repo https://prometheus-community.github.io/helm-charts \ + prometheus-agent kube-prometheus-stack + ``` + +2. **Enable Metrics in GXCache**: + + ```bash + helm upgrade gxcache -n gxcache . \ + --values values.yaml \ + --set webhook.metrics.enabled=true \ + --set service.metrics.enabled=true + ``` + +3. **Access Prometheus**: + + ```bash + kubectl port-forward -n monitoring svc/prometheus-operated 9090:9090 + ``` + +### Logging + +GXCache logs include: + +- Cache hit/miss events +- Shader compilation requests +- Error conditions and troubleshooting information +- Performance metrics + +## Troubleshooting + +### Common Issues + +**Webhook Not Working** + +- Verify webhook is running and healthy +- Check webhook configuration and secrets +- Ensure proper RBAC permissions + +**Cache Not Storing Shaders** + +- Verify GXCache service is running +- Check storage configuration and persistent volumes +- Review application logs for shader compilation errors + +**Low Cache Hit Ratio** + +- Review cache size configuration +- Check cache eviction policies +- Monitor storage performance + +**Storage Issues** + +- Verify storage class availability +- Check persistent volume claims +- Monitor disk space usage + +## Best Practices + +### Deployment + +- Deploy GXCache before deploying GPU workloads +- Use high-performance storage for better cache performance +- Monitor cache performance and adjust configuration as needed + +### Configuration + +- Size cache appropriately for your shader workload +- Use high-performance storage for better performance +- Enable metrics for monitoring and optimization + +### Security + +- Use proper RBAC permissions for webhook and service +- Secure communication between components +- Regularly update GXCache images for security patches + +### Maintenance + +- Monitor cache hit ratios and adjust cache size as needed +- Regularly review and clean up unused cached shaders +- Update GXCache images regularly for security patches +- Monitor storage usage and plan for growth diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/index.md b/docs/self-managed-1.0/cluster-management/index.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/index.md rename to docs/self-managed-1.0/cluster-management/index.md diff --git a/docs/self-managed-1.0/cluster-management/kai-scheduler.md b/docs/self-managed-1.0/cluster-management/kai-scheduler.md new file mode 100644 index 0000000000..3443c3f766 --- /dev/null +++ b/docs/self-managed-1.0/cluster-management/kai-scheduler.md @@ -0,0 +1,99 @@ +# KAI Scheduler Integration Guide + +[KAI Scheduler](https://github.com/kai-scheduler/KAI-Scheduler) is an open +source Kubernetes scheduler for AI workloads. NVCF uses it for GPU bin-packing, +queues, gang scheduling, and topology-aware placement. + +## Install KAI Scheduler + + +Use a tested [KAI Scheduler release](https://github.com/kai-scheduler/KAI-Scheduler/releases) +that is compatible with the NVCF compute plane stack. + + +Set `addons.kaiScheduler.enabled` in the `nvcf-compute-plane` Helmfile +environment to install KAI Scheduler as release and namespace `kai-scheduler`. +Grove, Dynamo, and topology-aware scheduling require this add-on. Skip the +manual installation below when the add-on is enabled. + +Use the manual path when KAI is managed outside the compute plane stack. + +NVCA expects a parent queue named `default-parent-queue` and a child queue +named `default-queue`. Other queues may also exist. + + +Set unlimited (`-1`) quotas and limits on every queue used for NVCF workloads. +This lets NVCA track the complete cluster capacity. If NVCF and non-NVCF +workloads share a cluster with limited KAI queues, enable +[Shared Cluster mode](./configuration.md#cluster-features) so NVCA excludes +non-NVCF nodes from capacity tracking and scheduling. + + +Create `values.yaml` with the required default queues: + + + +```yaml title="kai-scheduler-queues.yaml" +scheduler: + placementStrategy: binpack + plugins: + nodeplacement: + arguments: + gpu: binpack + cpu: spread + actions: + preempt: + enabled: false + consolidation: + enabled: false + +defaultQueue: + createDefaultQueue: true + parentName: default-parent-queue + childName: default-queue + parentResources: + cpu: + quota: -1 + limit: -1 + overQuotaWeight: 1 + gpu: + quota: -1 + limit: -1 + overQuotaWeight: 1 + memory: + quota: -1 + limit: -1 + overQuotaWeight: 1 + childResources: + cpu: + quota: -1 + limit: -1 + overQuotaWeight: 1 + gpu: + quota: -1 + limit: -1 + overQuotaWeight: 1 + memory: + quota: -1 + limit: -1 + overQuotaWeight: 1 + +``` + + + +```bash +helm install kai-scheduler oci://ghcr.io/kai-scheduler/kai-scheduler/kai-scheduler -f values.yaml -n kai-scheduler --create-namespace --version v0.14.0 +``` + +## Schedule multi-Pod workloads + +KAI can hold a multi-Pod workload until all required members fit. Grove and +Dynamo build on this behavior for multi-role inference services. See +[Gang Scheduling](./gang-scheduling.md) for add-on configuration, workload +examples, supported resource types, and troubleshooting. + +On NVLink-optimized clusters, KAI can also place the complete gang in one GPU +clique. See +[Topology-Aware Scheduling](./topology-aware-scheduling.md) for GPU DRA +prerequisites, topology configuration, Grove bindings, and function examples. diff --git a/docs/self-managed-1.0/cluster-management/model-cache.md b/docs/self-managed-1.0/cluster-management/model-cache.md new file mode 100644 index 0000000000..4fae32925c --- /dev/null +++ b/docs/self-managed-1.0/cluster-management/model-cache.md @@ -0,0 +1,201 @@ +# Helm Model Caching + +The NVIDIA Cloud Functions Agent (NVCA) can cache model and resource artifacts +for Helm-based functions and tasks. The cache works like shared storage for +large model and resource files. The first workload downloads an artifact set, +and later workloads can mount the populated cache instead of downloading +another copy. + +This page describes the Helm model cache behavior in NVCA 3.2. + +## Before You Begin + +Model caching requires both of these conditions: + +- The function or task includes model or resource artifacts. +- The `CachingSupport` cluster feature is enabled. See + [Caching Support](./configuration.md#caching-support). + +Durable reuse across workloads also requires a shared storage backend. If no +shared backend is available, NVCA uses a pod-local `emptyDir` volume. See +[Backend Selection](#backend-selection). + +For the NVCA-managed Samba backend, the cluster must also have: + +- The `nvcf-sc` StorageClass. +- The SMB CSI driver. +- The `HelmSharedStorage` feature flag enabled. See + [Enable Helm Shared Storage](./configuration.md#enable-helm-shared-storage). + +## How the Cache Works + +For a requested cache, the control plane sends NVCA a +`CacheLaunchSpecification` with `cacheArtifacts=true`, a positive `cacheSize`, +and a `cacheHandle`. The handle identifies the complete artifact set. Functions +or tasks that request the same artifacts should receive the same handle. + +For a durable backend, the flow is: + +```text +Function A ----> writer Job ----> populated cache for one cacheHandle + | +Function A <---- read-only attachment ----+ +Function B <---- read-only attachment ----+ +``` + +1. NVCA selects the first available storage backend. +1. NVCA creates a model-cache `StorageRequest` in the workload namespace. +1. A Kubernetes Lease selects one request as the writer for the cache handle. +1. NVCA creates one writer Job in the `nvca-modelcache-init` namespace. +1. The writer downloads the artifacts to writable storage and exits. +1. NVCA records durable cluster state that marks the cache as populated. +1. NVCA creates a namespace-local, read-only attachment for the workload. +1. A later request with the same cache handle reuses the populated cache and + skips the writer Job. + +The durable object and its populated marker are the source of truth for reuse. +Metrics report cache activity, but they do not determine whether a cache is +ready. + +## Backend Selection + +NVCA uses the first matching backend in this order: + +| Priority | Cluster condition | Backend | Reuse behavior | +| --- | --- | --- | --- | +| 1 | The model cache StorageClass (`nvcf-sc` unless overridden) is provisioned by `nvmesh-csi.excelero.com` | NVMesh | Durable reuse across namespaces | +| 2 | `nvcf-miniservice-sc` exists | Shared filesystem on the model cache StorageClass | Durable reuse across namespaces | +| 3 | `HelmSharedStorage` is enabled | NVCA-managed Samba | Durable reuse across namespaces | +| 4 | No shared backend is available | `emptyDir` | Pod-local caching only | + +NVCA does not create `nvcf-miniservice-sc`. Its presence only tells NVCA that +the cluster has a shared filesystem; the cache itself is written to the model +cache StorageClass (`nvcf-sc` unless overridden), and readers are derived from +the writer's volume rather than provisioned separately. A provisioner that creates an +isolated directory, access point, or subvolume for every claim does not provide +cross-namespace reuse through this backend. + +The Samba backend creates a separate Samba server and a backing volume on the +model cache StorageClass (`nvcf-sc` unless overridden) for each cache handle. Readers mount the same SMB share with read-only +credentials. + +## Workload Mounts + +NVCA adds one `model-data` volume to the Helm workload and mounts it at: + +- `/config/models` +- `/config/resources` + +For durable backends, each workload namespace receives its own reader claim or +attachment. The workload mounts the cache read-only. The attachment mechanism +depends on the backend, but the paths inside the workload do not change: + +- NVMesh readers use namespace-specific secondary volume handles derived from + the same primary volume. +- Shared-filesystem readers use separate claims that expose the same cached + data. +- Samba readers mount the same SMB share. + +## Pod-Local Fallback + +When no durable shared backend is available, each pod receives: + +- A writable `emptyDir` volume. +- A `model-cache-init` init container that downloads the artifacts. +- The same model and resource mounts used by durable backends. + +This fallback makes the artifacts available during that pod's startup. It does +not reuse downloaded data across pods or namespaces. + +The `nvcf-miniservice-metadata` ConfigMap carries the init-container environment +to the NVCA webhook. The ConfigMap is transport metadata. Its presence does not +enable caching for a workload. + +## Workloads Without Cached Artifacts + +Enabling `CachingSupport` does not make every Helm workload use a model cache. +When `cacheArtifacts=false`, NVCA does not add: + +- A model-cache `StorageRequest`. +- A `model-data` volume or mount. +- A model-cache init container. +- Model-cache init metadata. + +An unused cache handle or a rounded cache size in the launch data does not, by +itself, mean that the workload requested caching. + +## Failure Behavior + +Model caching is best-effort. If the writer Job or a storage operation fails, +NVCA marks the model-cache `StorageRequest` as failed and reports the +`CacheSuccessful=False` MiniService condition with the `CachingFailed` reason. +NVCA then stops waiting for the cache and installs the Helm workload without the +cache attachment. + +A cache failure can increase startup time or leave cached artifacts unavailable, +but it does not block installation of the Helm workload. + +## Verify Model Cache Resources + +List model-cache requests across workload namespaces: + +```bash +kubectl get storagerequests.nvca.nvcf.nvidia.io --all-namespaces +``` + +Inspect the writer, coordination, and durable storage resources: + +```bash +kubectl get jobs,leases,pvc -n nvca-modelcache-init +``` + +For each workload namespace, list the cache backend and the namespace-local +reader PVC: + +```bash +workload_namespace="" + +kubectl get storagerequests.nvca.nvcf.nvidia.io \ + -n "$workload_namespace" \ + -o custom-columns='NAME:.metadata.name,TYPE:.spec.type,PHASE:.status.phase,BACKEND:.spec.modelCache.backend,READER-PVC:.status.modelCache.readOnlyPVCName' +``` + +For a `modelcache` row with a durable backend, confirm that the phase is `Ready` +and that `READER-PVC` contains a claim name. Verify that the claim is bound: + +```bash +reader_pvc="" +kubectl get pvc "$reader_pvc" -n "$workload_namespace" -o wide +``` + +Inspect the workload's `model-data` volume. A durable attachment returns the +claim name followed by `true`: + +```bash +pod_name="" + +kubectl get pod "$pod_name" -n "$workload_namespace" \ + -o jsonpath='{.spec.volumes[?(@.name=="model-data")].persistentVolumeClaim.claimName}{"\t"}{.spec.volumes[?(@.name=="model-data")].persistentVolumeClaim.readOnly}{"\n"}' +``` + +Inspect the PersistentVolume (PV) bound to the claim when you need to confirm +the backend-specific attachment: + +```bash +persistent_volume="$(kubectl get pvc "$reader_pvc" \ + -n "$workload_namespace" -o jsonpath='{.spec.volumeName}')" + +kubectl get pv "$persistent_volume" -o yaml +``` + +The PV identifies the attachment type: + +- An NVMesh reader has a namespace-specific CSI volume handle derived from the + primary volume. +- A Samba reader uses the `smb.csi.k8s.io` driver and points to the cache + handle's SMB share. +- A shared-filesystem reader is a static PV bound to the writer's volume by name, with no StorageClass; only the writer claim uses the model cache StorageClass. + +Repeat these checks for every workload namespace. Later workloads with the same +cache handle should receive their own read-only attachment without creating +another writer Job. diff --git a/docs/self-managed-1.0/cluster-management/monitoring.md b/docs/self-managed-1.0/cluster-management/monitoring.md new file mode 100644 index 0000000000..02d33205d9 --- /dev/null +++ b/docs/self-managed-1.0/cluster-management/monitoring.md @@ -0,0 +1,122 @@ +# Monitoring & Observability + +The NVIDIA Cluster Agent and Operator provide built-in monitoring through Prometheus metrics, +structured logging, and OpenTelemetry tracing. + +## Monitoring Data + +### Metrics + +**Prerequisites** + +To use the PodMonitor and ServiceMonitor examples below, you must first install the Prometheus Operator. Follow the [Prometheus Operator installation guide](https://prometheus-operator.dev/docs/getting-started/installation/) to set this up in your cluster. + +The cluster agent and operator emit Prometheus-style metrics. The following metric labels are available by default. The full list of available metrics are updated regularly and therefore not listed. + +| Metric Label | Metric Label Description | +| ------------------ | -------------------------------- | +| nvca_event_name | The name of the event | +| nvca_nca_id | The NCA ID of this NVCA instance | +| nvca_cluster_name | The NVCA cluster name | +| nvca_cluster_group | The NVCA cluster group | +| nvca_version | The NVCA version | + +Cluster maintainers can scrape the available metrics. See a full example of how to do this with an OpenTelemetry Collector in the cluster [cluster monitoring example](https://github.com/NVIDIA/nvcf/tree/main/examples/cluster-monitoring-sample). + +Use the following examples of a PodMonitor for NVCA Operator and ServiceMonitor for NVCA for reference: + +**Sample NVCA Operator PodMonitor** + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + labels: + app.kubernetes.io/component: metrics + app.kubernetes.io/instance: prometheus-agent + app.kubernetes.io/name: metrics-nvca-operator + jobLabel: metrics-nvca-operator + release: prometheus-agent + prometheus.agent/podmonitor-discover: "true" + name: metrics-nvca-operator + namespace: monitoring +spec: + podMetricsEndpoints: + - port: http + scheme: http + path: /metrics + jobLabel: jobLabel + selector: + matchLabels: + app.kubernetes.io/name: nvca-operator + namespaceSelector: + matchNames: + - nvca-operator +``` + +**Sample NVCA ServiceMonitor** + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/component: metrics + app.kubernetes.io/instance: prometheus-agent + app.kubernetes.io/name: metrics-nvca + jobLabel: metrics-nvca + release: prometheus-agent + prometheus.agent/servicemonitor-discover: "true" + name: prometheus-agent-nvca + namespace: monitoring +spec: + endpoints: + - port: nvca + jobLabel: jobLabel + selector: + matchLabels: + app.kubernetes.io/name: nvca + namespaceSelector: + matchNames: + - nvca-system +``` + +### Logs + +Both the Cluster Agent and Cluster Agent Operator emit logs locally by default. + +Local logs for the NVIDIA Cluster Agent Operator can be obtained via `kubectl`: + +```bash +kubectl logs -l app.kubernetes.io/instance=nvca-operator -n nvca-operator --tail 20 +``` + +Similarly, NVIDIA Cluster Agent logs can be obtained with the following command via kubectl: + +```bash +kubectl logs -l app.kubernetes.io/instance=nvca -n nvca-system --tail 20 +``` + + +Current function-level inference container logs are **not supported** for functions deployed on non-NVIDIA-managed clusters. Customers are encouraged to emit logs directly from their inference containers running on their own clusters to any third-party tool, there are no public egress limitations for containers. + + + +### Tracing + +The NVIDIA Cluster Agent provides OpenTelemetry integration for exporting traces and events to compatible collectors. As of agent version 2.0, the only supported collector receiver is Lightstep. + +**Enable Tracing with Lightstep** + +1. Get your [Lightstep access token](https://docs.lightstep.com/docs/create-and-manage-access-tokens) from the [Lightstep UI](https://app.lightstep.com) and set to `LS_ACCESS_TOKEN` environment variable. +2. Get the NVCF cluster name: + +```bash +nvcf_cluster_name="$(kubectl get nvcfbackends -n nvca-operator -o name | cut -d'/' -f2)" +``` + +1. Apply the tracing configuration: + +```bash +kubectl patch nvcfbackends.nvcf.nvidia.io -n nvca-operator "$nvcf_cluster_name" --type=merge --patch="{\"spec\":{\"overrides\":{\"featureGate\":{\"otelConfig\":{\"exporter\":\"lightstep\",\"serviceName\":\"nvcf-nvca\",\"accessToken\":\"${LS_ACCESS_TOKEN}\"}}}}}" +``` diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/multi-tenancy.md b/docs/self-managed-1.0/cluster-management/multi-tenancy.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/multi-tenancy.md rename to docs/self-managed-1.0/cluster-management/multi-tenancy.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/nsight-profiling.md b/docs/self-managed-1.0/cluster-management/nsight-profiling.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/nsight-profiling.md rename to docs/self-managed-1.0/cluster-management/nsight-profiling.md diff --git a/docs/self-managed-1.0/cluster-management/reference.md b/docs/self-managed-1.0/cluster-management/reference.md new file mode 100644 index 0000000000..142e5f5a64 --- /dev/null +++ b/docs/self-managed-1.0/cluster-management/reference.md @@ -0,0 +1,248 @@ +# Helm Values Reference + +The `nvca-operator` Helm chart is configured through a standard Helm values file (`values.yaml`) +passed to `helm upgrade -f values.yaml`. This page documents all available parameters. + + +The parameters listed below are a snapshot and may not reflect the latest chart version. +Always refer to the `values.yaml` and `values.schema.json` included in your chart +version for the authoritative list of parameters and defaults: + + + +## How Values Are Structured + +The chart values are organized into two layers: + +1. **Shared parameters** (top-level) — These control the operator image, authentication, + node placement, network policies, observability, agent resources, and agent runtime config. + Examples: `image`, `ngcConfig`, `nodeSelector`, `networkPolicy`, `agent`, + `agentConfig`. +2. **\`\`selfManaged.\*\`\`** — Used when `ngcConfig.clusterSource` is `"self-managed"`. + These define the backend configuration including NVCA version, feature gates, cluster + attributes, and manual GPU config. + +**The key field is \`\`ngcConfig.clusterSource\`\`:** + +- `"self-managed"` — The operator reads backend configuration from `selfManaged.*` values. + +```yaml +# --- Shared parameters (all modes) --- +image: + repository: "nvcr.io/nvidia/nvcf-byoc/nvca-operator" +ngcConfig: + clusterSource: "helm-managed" # ← This determines which section below is used + serviceKey: "" +nodeSelector: + key: "node.kubernetes.io/instance-type" + value: "m5.2xlarge" + +# --- Only read when clusterSource is "helm-managed" --- +helmManaged: + cloudProvider: "aws" + clusterRegion: "us-west-2" + nvcaVersion: "2.97.0" + featureGateValues: ["DynamicGPUDiscovery"] + +# --- Only read when clusterSource is "self-managed" --- +# selfManaged: +# nvcaVersion: "2.97.0" +# featureGateValues: ["DynamicGPUDiscovery"] +``` + + +The `helmManaged` and `selfManaged` sections share many of the same fields +(`nvcaVersion`, `featureGateValues`, `gpuManualInstanceConfigB64`, +`clusterAttributes`). The difference is that `helmManaged` also requires cluster +identity fields (`cloudProvider`, `clusterRegion`, `clusterGroupID`, +`clusterGroupName`) that self-managed deployments get from their own control plane. + + + +## Shared Parameters + +These parameters apply to all deployment modes. + +### NVCA Operator + +```yaml +## Container images +image: + repository: "nvcr.io/nvidia/nvcf-byoc/nvca-operator" # Operator image path + tag: "" # Defaults to chart version + pullPolicy: IfNotPresent + +nvcaImage: + repositoryOverride: "" # Override NVCA agent image path (staging/testing only) + pullPolicy: IfNotPresent + +## Image pull secrets +generateImagePullSecret: true # Auto-generate from ngcConfig.serviceKey +imagePullSecretName: "nvca-operator-image-pull" # Name of the generated secret +imagePullSecrets: [] # Additional pre-existing pull secrets + +## Service account +serviceAccount: + create: true + annotations: {} + name: "" # Auto-generated if empty + +## Operator settings +replicaCount: 1 +systemNamespace: nvca-operator +logLevel: info # debug, info, warn, error +priorityClassName: "" # K8s PriorityClassName for eviction preference +k8sVersionOverride: "" # Override K8s version NVCA registers with +enableGXCache: true # Enable GXCache support +ddcsIPAllowList: "" # Comma-separated CIDRs for DDCS access control +nvcaHelmRepositoryPrefix: "" # Restrict Helm repos to specific org/team +``` + +### NGC Authentication + +```yaml +ngcConfig: + username: '$oauthtoken' + serviceKey: "" # NGC Cluster Key or NVCF API Key (NAK) + serviceKeySecretName: "ngc-service-key" # K8s Secret name if serviceKey not set inline + serviceKeySecretKeyName: "ngcServiceKey" # Key within the Secret + apiURL: https://api.ngc.nvidia.com # NGC API URL (override for self-hosted) + clusterSource: ngc-managed # "ngc-managed", "helm-managed", or "self-managed" +``` + +### Node Selector + +```yaml +nodeSelector: + key: node.kubernetes.io/instance-type # Label key for operator pod placement + value: "" # Label value (empty = no constraint) +``` + +### Network Policies + +```yaml +networkPolicy: + clusterNetworkCIDRs: # CIDRs that workload pods are NOT allowed to access + - "10.0.0.0/8" + - "172.16.0.0/12" + - "192.168.0.0/16" + - "100.64.0.0/12" + customPolicies: [] # Custom NetworkPolicy definitions for function namespaces +``` + +### OpenTelemetry + +```yaml +otel: + enabled: false + lightstep: + serviceName: "" # Lightstep service name + accessToken: "" # Lightstep API token +``` + +### Agent Configuration + +```yaml +agent: + cacheMountOptionsEnabled: true + cacheMountOptions: "ro,norecovery,nouuid" + workerDegradationPeriod: "" # e.g., "90m", "1h30m" + secretMirrorNamespace: nvca-operator # Namespace to mirror custom secrets from + secretMirrorLabelSelector: "" # Label selector for mirrored secrets + customAnnotations: {} # Extra annotations on the agent pod + functionEnvOverrides: {} # Override infra container images for functions + taskEnvOverrides: {} # Override infra container images for tasks + overrideEnvironmentVariables: {} # Override env vars on the NVCA agent container + resources: + limits: + cpu: 1000m + memory: 4Gi + requests: + cpu: 100m + memory: 200Mi + +## Merge custom YAML into the generated NVCA agent config at runtime +agentConfig: + mergeConfig: "" + # Example: + # mergeConfig: | + # agent: + # logLevel: debug + +## OTel Collector sidecar (for K8s event collection) +otelCollector: + enabled: false + imageRepository: "" # Auto-calculated if empty + imageTag: 0.143.2 +``` + +## Helm-Managed Parameters + +Only used when `ngcConfig.clusterSource: "helm-managed"`. + +```yaml +helmManaged: + ## Cluster identity (REQUIRED, immutable after initial registration) + cloudProvider: "" # e.g., "aws", "gcp", "azure", "ON-PREM", "NCP" + clusterRegion: "" # e.g., "us-west-2" + clusterGroupID: "" # Unique cluster group identifier + clusterGroupName: "" # Human-readable cluster group name + + ## Backend configuration + nvcaVersion: "" # NVCA agent version (REQUIRED) + clusterDescription: "" # Defaults to cluster name if empty + featureGateValues: [] # e.g., ["DynamicGPUDiscovery", "CachingSupport"] + gpuManualInstanceConfigB64: "" # Base64-encoded GPU config (manual instance only) + clusterAttributes: [] # e.g., ["CacheOptimized", "NVLinkOptimized"] + + ## Authentication (optional) + oAuthClientID: "" # OAuth2/OIDC client ID (for internal NVIDIA clusters) + oAuthClientSecretKey: "" # Secret key for OAuth client + + ## Image overrides (advanced, usually auto-calculated) + imageCredHelper: + imageRepository: "" + imageTag: 0.11.1 + otelCollector: + imageRepository: "" + imageTag: 0.143.2 +``` + +## Self-Managed Parameters + +Only used when `ngcConfig.clusterSource: "self-managed"` (self-hosted NVCF). + +```yaml +selfManaged: + nvcaVersion: "" # NVCA agent version (REQUIRED) + featureGateValues: ["DynamicGPUDiscovery"] # Default includes GPU discovery + gpuManualInstanceConfigB64: "" # Base64-encoded GPU config (manual instance only) + clusterAttributes: [] # e.g., ["CacheOptimized"] + + ## Image overrides (advanced, usually auto-calculated) + imageCredHelper: + imageRepository: "" + imageTag: 0.11.1 + otelCollector: + imageRepository: "" + imageTag: 0.143.2 +``` + + +The `selfManaged` and `helmManaged` sections share the same backend fields. The key +differences are: + +- `selfManaged` does not have cluster identity fields (`cloudProvider`, + `clusterRegion`, `clusterGroupID`, `clusterGroupName`) — these come from the + self-hosted control plane. +- `selfManaged.featureGateValues` defaults to `["DynamicGPUDiscovery"]`. To disable a + feature, remove it from the list (set to `[]`). +- `helmManaged.featureGateValues` defaults to `[]`. To disable a feature, prefix it + with `-` (e.g., `["-DynamicGPUDiscovery"]`). + + + +## Related Documentation + +- [NVCA Configuration](./configuration.md) — how to use these values to configure specific features (caching, network policies, manual instance config, etc.) +- [Agent config merging](./configuration.md) — using `agentConfig.mergeConfig` for runtime config overrides diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/self-managed.md b/docs/self-managed-1.0/cluster-management/self-managed.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/self-managed.md rename to docs/self-managed-1.0/cluster-management/self-managed.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/topology-aware-scheduling.md b/docs/self-managed-1.0/cluster-management/topology-aware-scheduling.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/topology-aware-scheduling.md rename to docs/self-managed-1.0/cluster-management/topology-aware-scheduling.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/configure-autoscaling.md b/docs/self-managed-1.0/configure-autoscaling.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/configure-autoscaling.md rename to docs/self-managed-1.0/configure-autoscaling.md diff --git a/docs/user/container-functions.md b/docs/self-managed-1.0/container-functions.md similarity index 100% rename from docs/user/container-functions.md rename to docs/self-managed-1.0/container-functions.md diff --git a/docs/user/container-tasks.md b/docs/self-managed-1.0/container-tasks.md similarity index 100% rename from docs/user/container-tasks.md rename to docs/self-managed-1.0/container-tasks.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/control-plane-installation.md b/docs/self-managed-1.0/control-plane-installation.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/control-plane-installation.md rename to docs/self-managed-1.0/control-plane-installation.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/control-plane-operations.md b/docs/self-managed-1.0/control-plane-operations.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/control-plane-operations.md rename to docs/self-managed-1.0/control-plane-operations.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/csp-end-to-end-example-installation.md b/docs/self-managed-1.0/csp-end-to-end-example-installation.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/csp-end-to-end-example-installation.md rename to docs/self-managed-1.0/csp-end-to-end-example-installation.md diff --git a/docs/self-managed-1.0/example-dashboards.md b/docs/self-managed-1.0/example-dashboards.md new file mode 100644 index 0000000000..0deb7dd9fd --- /dev/null +++ b/docs/self-managed-1.0/example-dashboards.md @@ -0,0 +1,356 @@ +# Example Dashboards Deployment + + +**These Helm charts are provided as examples only and are not intended for production use.** + +The `nvcf-observability-reference-stack` and `nvcf-example-dashboards` Helm charts are +designed to help users understand useful metrics, logs, and traces, and to serve as +inspiration for creating custom dashboards tailored to their observability needs. + +**Important limitations:** + +- No security hardening +- No SSL/TLS encryption +- No authentication or authorization +- No support for production workloads +- Not supported by NVIDIA for uses beyond example and reference purposes + +For production deployments, users should integrate with their own observability infrastructure +following the guidance in [self-hosted-observability](./observability.md). + + + +## Overview + +This guide provides step-by-step instructions for deploying the NVCF observability reference +stack and example dashboards in a local or development environment. These example components +demonstrate how to collect and visualize metrics, logs, and traces from a self-hosted NVCF deployment. + +The example stack includes: + +- **nvcf-observability-reference-stack**: A reference implementation of an observability backend + (Prometheus, Grafana, Loki, Tempo, and OpenTelemetry Collector) +- **nvcf-example-dashboards**: Pre-configured Grafana dashboards showing key NVCF control-plane metrics + + +The observability reference stack supports a single-cluster deployment or the +control-plane cluster in a split-topology deployment. A separate observability +reference stack for an individual GPU cluster is not currently supported. + + + +## Prerequisites + +Before deploying the example dashboards, you need: + +1. A Kubernetes cluster +2. Self-hosted NVCF control-plane deployed and running +3. `helm` CLI installed +4. `kubectl` configured to access your cluster + + +To populate Prometheus-backed dashboards, deploy the control-plane with +`global.observability.metrics.enabled` set to `true`. + +To view traces in the example stack, the control-plane must be deployed +with tracing enabled and configured to send OTLP traces to the +observability collector. See [Tracing Configuration](./observability.md) +for the required Helm overrides under `global.observability.tracing` (e.g., +`enabled`, `collectorEndpoint`, `collectorPort`, +`collectorProtocol`). + + + +### Enabling metrics and tracing after initial deployment + +If observability was not configured during the initial self-hosted NVCF cluster +deployment, you can enable metrics later so that control-plane services expose +Prometheus metrics. Enable tracing if you also want the control-plane to send +OTLP traces to the deployed observability collector. Use the following steps. + +1. Edit the environment file (`environments/.yaml`) to + enable metrics and tracing. Set `collectorEndpoint` to the deployed + collector's service address. For the example observability reference stack, + the collector runs in the `observability` namespace. Example: + + ```yaml + global: + observability: + metrics: + enabled: true + tracing: + enabled: true + collectorEndpoint: "otel-collector-gateway-collector.observability.svc.cluster.local" + collectorPort: 4317 + collectorProtocol: http + ``` + +2. Apply the configuration changes from your control-plane deployment directory: + + ```bash + HELMFILE_ENV= helmfile sync + ``` + + Replace `` with your environment name (e.g., `eks-example`). + +## Deployment Steps + +### Step 1: Install the Observability Reference Stack + +Once your self-managed NVCF control-plane is up and running, install the observability +reference stack: + +```bash +# Add and update the public NVCF Helm repository +helm repo add nvcf https://helm.ngc.nvidia.com/nvidia/nvcf --force-update +helm repo update + +# Check NGC for the latest version of this Helm chart: +# https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nvcf/helm-charts/nvcf-observability-reference-stack + +helm upgrade \ + --install observability \ + nvcf/nvcf-observability-reference-stack \ + --version 1.10.0 \ + --namespace observability \ + --create-namespace +``` + +For a split-topology deployment, run the command against the control-plane +cluster and disable the NVCA ServiceMonitor. NVCA runs on the GPU cluster, so +the control-plane observability stack does not have an NVCA scrape target. + +```bash +helm upgrade \ + --install observability \ + nvcf/nvcf-observability-reference-stack \ + --version 1.10.0 \ + --namespace observability \ + --create-namespace \ + --set nvcfServiceMonitors.nvcaEnabled=false +``` + +Do not install this reference stack on the GPU cluster. GPU cluster-specific +observability stack support is not currently available. + +This will deploy: + +- Prometheus for metrics collection +- Grafana for visualization +- Loki for log aggregation +- Tempo for distributed tracing +- OpenTelemetry Collector for telemetry processing +- Fluent Bit for log collection + +The OTel Collector and Fluent Bit cluster-scoped config are disabled in this +initial install; they are enabled in Step 2. + + +The Grafana installation in the observability reference stack includes the +`ae3e-plotly-panel` plugin for use with the example dashboards. + + + +**Verify the deployment:** + +```bash +# Check that all pods are running +kubectl get pods -n observability + +# Wait for all pods to be Ready +kubectl wait --for=condition=ready pod --all -n observability --timeout=300s +``` + +### Step 2: Enable the OTel Collector and Fluent Bit Cluster Config + +The initial install does not enable the OTel Collector or the Fluent Bit +cluster-scoped config by default. A second Helm upgrade is required to set +`otel-collector.enabled=true` and `fluentBitClusterConfig.enabled=true` so +that the `observability-gateway-collector` service (and related gateway +services) are deployed and Fluent Bit cluster resources (ClusterFilter, +ClusterOutput, ClusterFluentBitConfig) are managed by Helm. This has to be +split into separate installs due to a chicken and egg problem with the CRDs. +These services are needed for the control-plane to send OTLP traces to Tempo. + +Run the following upgrade (use the same chart version and namespace as in Step 1): + +For split-topology deployments, `--reuse-values` preserves +`nvcfServiceMonitors.nvcaEnabled=false` from the initial installation. + +```bash +helm upgrade observability \ + nvcf/nvcf-observability-reference-stack \ + --version 1.10.0 \ + --namespace observability \ + --reuse-values \ + --set fluentBitClusterConfig.enabled=true \ + --set otel-collector.enabled=true \ + --wait \ + --timeout 5m +``` + +**Verify the OTel Collector is running:** + +```bash +kubectl get pods -n observability | grep observability-gateway +kubectl get svc -n observability | grep observability-gateway +``` + +You should see the `observability-gateway-collector` pod and the +`observability-gateway-collector` service (and related gateway services). + +### Step 3: Install the Example Dashboards + +Once the observability reference stack is deployed and the OTel Collector is +enabled, install the example dashboards: + +```bash +# Check NGC for the latest version of this Helm chart: +# https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nvcf/helm-charts/nvcf-example-dashboards + +helm upgrade \ + --install nvcf-example-dashboards \ + nvcf/nvcf-example-dashboards \ + --version 1.6.0 \ + --namespace observability \ + --create-namespace +``` + +This will configure Grafana with pre-built dashboards for: + +- NVCF API +- Invocation Service +- SPOT Instance Service (SIS) +- Encrypted Secrets Service (ESS) +- Cassandra +- Vault +- Worker Pods (Utils, Init, and Inference containers) + +**Access Grafana:** + +```bash +# Port-forward to access Grafana UI +kubectl port-forward -n observability \ + svc/$(kubectl get svc -n observability -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}') \ + 3000:80 +``` + +Then open your browser to `http://localhost:3000` and log in to view the dashboards. + +### Step 4: Generate Dashboard Data + +To populate the dashboards with meaningful data, you need to deploy and invoke functions. +Deploy and invoke functions using your own commands or tools. The example dashboards will +automatically populate as your NVCF control-plane handles function requests. + +## Cleanup and Uninstallation + +When you're finished testing or want to remove the example observability stack, follow these +steps: + +### Step 1: Delete Custom Resources + +First, delete any custom resources created by the observability stack: + +```bash +# Delete FluentBit custom resources (namespace-scoped) +kubectl delete fluentbits.fluentbit.fluent.io --all -A + +# Delete OpenTelemetry Collector custom resources +kubectl delete opentelemetrycollectors.opentelemetry.io --all -A +``` + +If you enabled `fluentBitClusterConfig.enabled=true` in Step 2 of deployment, +the Fluent Bit cluster-scoped resources (ClusterFilter, ClusterOutput, +ClusterFluentBitConfig) are managed by Helm and will be removed when you +uninstall the Helm release in the next step. + +### Step 2: Uninstall Helm Releases + +Uninstall both Helm releases: + +```bash +# Uninstall example dashboards +helm uninstall nvcf-example-dashboards -n observability + +# Uninstall observability reference stack +helm uninstall observability -n observability +``` + +### Step 3: Delete the Namespace + +Finally, delete the observability namespace: + +```bash +# Delete the namespace (this will remove any remaining resources) +kubectl delete namespace observability +``` + + +If you deployed NVCF to a namespace other than `observability`, make sure to only delete +the observability namespace, not your NVCF control-plane namespace. + + + +## Troubleshooting + +**Pods not starting:** + +Check pod status and logs: + +```bash +kubectl get pods -n observability +kubectl describe pod -n observability +kubectl logs -n observability +``` + +**Dashboards not showing data:** + +1. Verify Prometheus is scraping metrics: + + ```bash + # Port-forward to Prometheus + kubectl port-forward -n observability svc/prometheus 9090:9090 + + # Check targets at http://localhost:9090/targets + ``` + +2. Verify NVCF services are exposing metrics: + + ```bash + # Port-forward to an NVCF service + kubectl port-forward -n nvcf svc/nvcf-api 8080:8080 + + # Curl the metrics endpoint + curl http://localhost:8080/metrics + ``` + +3. Check that ServiceMonitors are created: + + ```bash + kubectl get servicemonitor -n nvcf + ``` + +**Grafana login issues:** + +The default credentials are typically `admin/admin` or may be configured via Helm values. +Check the Helm chart documentation or values for the correct credentials. + +## Next Steps + +After exploring the example dashboards: + +1. Review the metrics, logs, and traces being collected +2. Identify which metrics are most relevant to your use case +3. Design and implement your own production-ready observability solution +4. Integrate with your existing enterprise observability platforms +5. Configure alerting based on your operational requirements + +For production deployments, see [self-hosted-observability](./observability.md) for guidance on integrating +with your own observability infrastructure. + +## Related Documentation + +- [self-hosted-observability](./observability.md): Production observability configuration +- [nvcf-observability-reference-stack on NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nvcf/helm-charts/nvcf-observability-reference-stack) +- [nvcf-example-dashboards on NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nvcf/helm-charts/nvcf-example-dashboards) diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/fake-gpu-operator.md b/docs/self-managed-1.0/fake-gpu-operator.md similarity index 98% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/fake-gpu-operator.md rename to docs/self-managed-1.0/fake-gpu-operator.md index 6a89569809..b56f1d119d 100644 --- a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/fake-gpu-operator.md +++ b/docs/self-managed-1.0/fake-gpu-operator.md @@ -138,7 +138,7 @@ The recommended sequence is: 1. Install KWOK 2. Install fake-gpu-operator and label target nodes 3. Verify `nvidia.com/gpu` appears in node allocatable resources -4. Proceed with the [control-plane installation](../user/helmfile-installation.md) +4. Proceed with the [control-plane installation](./helmfile-installation.md) ### If Installed After the Control Plane @@ -160,7 +160,7 @@ kubectl rollout status deployment nvca-operator -n nvca-operator --timeout=120s The operator restart will re-run the bootstrap init container, recreate the NVCFBackend resource, and spawn a fresh NVCA agent pod that discovers the simulated GPUs. -For details on the bootstrap process, see [Self-Managed Clusters](../user/cluster-management/self-managed.md) (Manual Cluster +For details on the bootstrap process, see [Self-Managed Clusters](./cluster-management/self-managed.md) (Manual Cluster Registration). ## Customization diff --git a/docs/user/function-creation.md b/docs/self-managed-1.0/function-creation.md similarity index 100% rename from docs/user/function-creation.md rename to docs/self-managed-1.0/function-creation.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/gateway-routing.md b/docs/self-managed-1.0/gateway-routing.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/gateway-routing.md rename to docs/self-managed-1.0/gateway-routing.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/generic-http-function-invocation.md b/docs/self-managed-1.0/generic-http-function-invocation.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/generic-http-function-invocation.md rename to docs/self-managed-1.0/generic-http-function-invocation.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/grpc-function-invocation.md b/docs/self-managed-1.0/grpc-function-invocation.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/grpc-function-invocation.md rename to docs/self-managed-1.0/grpc-function-invocation.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/grpc-invocation-enablement.md b/docs/self-managed-1.0/grpc-invocation-enablement.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/grpc-invocation-enablement.md rename to docs/self-managed-1.0/grpc-invocation-enablement.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/grpc-load-test-sli-guide.md b/docs/self-managed-1.0/grpc-load-test-sli-guide.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/grpc-load-test-sli-guide.md rename to docs/self-managed-1.0/grpc-load-test-sli-guide.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/grpc-load-testing.md b/docs/self-managed-1.0/grpc-load-testing.md similarity index 98% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/grpc-load-testing.md rename to docs/self-managed-1.0/grpc-load-testing.md index 785e7a5171..0fe00b389a 100644 --- a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/grpc-load-testing.md +++ b/docs/self-managed-1.0/grpc-load-testing.md @@ -5,8 +5,8 @@ ### Self-hosted CLI You need a working `nvcf-cli` configured against your self-managed cluster. -If you have not set this up yet, follow the [self-hosted-cli](../user/cli.md) guide to -install the binary and the [cli-configuration](../user/cli.md) section to point it at your +If you have not set this up yet, follow the [self-hosted-cli](./cli.md) guide to +install the binary and the [cli-configuration](./cli.md) section to point it at your gateway. Verify the CLI can reach the cluster before continuing: diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/helm-functions.md b/docs/self-managed-1.0/helm-functions.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/helm-functions.md rename to docs/self-managed-1.0/helm-functions.md diff --git a/docs/user/helm-tasks.md b/docs/self-managed-1.0/helm-tasks.md similarity index 100% rename from docs/user/helm-tasks.md rename to docs/self-managed-1.0/helm-tasks.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/helmfile-installation.md b/docs/self-managed-1.0/helmfile-installation.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/helmfile-installation.md rename to docs/self-managed-1.0/helmfile-installation.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/http-load-test-sli-guide.md b/docs/self-managed-1.0/http-load-test-sli-guide.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/http-load-test-sli-guide.md rename to docs/self-managed-1.0/http-load-test-sli-guide.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/http-load-testing.md b/docs/self-managed-1.0/http-load-testing.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/http-load-testing.md rename to docs/self-managed-1.0/http-load-testing.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/http-soak-testing.md b/docs/self-managed-1.0/http-soak-testing.md similarity index 98% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/http-soak-testing.md rename to docs/self-managed-1.0/http-soak-testing.md index df7e4b7272..b19f9eb4b2 100644 --- a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/http-soak-testing.md +++ b/docs/self-managed-1.0/http-soak-testing.md @@ -11,8 +11,8 @@ extended period (default 48 hours). ### Self-hosted CLI You need a working `nvcf-cli` configured against your self-managed cluster. -If you have not set this up yet, follow the [self-hosted-cli](../user/cli.md) guide to -install the binary and the [cli-configuration](../user/cli.md) section to point it at your +If you have not set this up yet, follow the [self-hosted-cli](./cli.md) guide to +install the binary and the [cli-configuration](./cli.md) section to point it at your gateway. Verify the CLI can reach the cluster before continuing: diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/image-mirroring.md b/docs/self-managed-1.0/image-mirroring.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/image-mirroring.md rename to docs/self-managed-1.0/image-mirroring.md diff --git a/docs/user/images/apipersonalkey.png b/docs/self-managed-1.0/images/apipersonalkey.png similarity index 100% rename from docs/user/images/apipersonalkey.png rename to docs/self-managed-1.0/images/apipersonalkey.png diff --git a/docs/user/images/grpc-reconnect-flow.png b/docs/self-managed-1.0/images/grpc-reconnect-flow.png similarity index 100% rename from docs/user/images/grpc-reconnect-flow.png rename to docs/self-managed-1.0/images/grpc-reconnect-flow.png diff --git a/docs/user/images/grpc-single-client.png b/docs/self-managed-1.0/images/grpc-single-client.png similarity index 100% rename from docs/user/images/grpc-single-client.png rename to docs/self-managed-1.0/images/grpc-single-client.png diff --git a/docs/user/images/lls-self-hosted-arch-streaming-proxy.png b/docs/self-managed-1.0/images/lls-self-hosted-arch-streaming-proxy.png similarity index 100% rename from docs/user/images/lls-self-hosted-arch-streaming-proxy.png rename to docs/self-managed-1.0/images/lls-self-hosted-arch-streaming-proxy.png diff --git a/docs/user/images/low-latency-streaming.png b/docs/self-managed-1.0/images/low-latency-streaming.png similarity index 100% rename from docs/user/images/low-latency-streaming.png rename to docs/self-managed-1.0/images/low-latency-streaming.png diff --git a/docs/user/images/nvcf-banner.svg b/docs/self-managed-1.0/images/nvcf-banner.svg similarity index 100% rename from docs/user/images/nvcf-banner.svg rename to docs/self-managed-1.0/images/nvcf-banner.svg diff --git a/docs/user/images/nvcf-gpu-cluster-targeting.svg b/docs/self-managed-1.0/images/nvcf-gpu-cluster-targeting.svg similarity index 100% rename from docs/user/images/nvcf-gpu-cluster-targeting.svg rename to docs/self-managed-1.0/images/nvcf-gpu-cluster-targeting.svg diff --git a/docs/user/images/nvcf-grpc-invocation-path.svg b/docs/self-managed-1.0/images/nvcf-grpc-invocation-path.svg similarity index 100% rename from docs/user/images/nvcf-grpc-invocation-path.svg rename to docs/self-managed-1.0/images/nvcf-grpc-invocation-path.svg diff --git a/docs/user/images/nvcf-grpc-multicluster-invocation.svg b/docs/self-managed-1.0/images/nvcf-grpc-multicluster-invocation.svg similarity index 100% rename from docs/user/images/nvcf-grpc-multicluster-invocation.svg rename to docs/self-managed-1.0/images/nvcf-grpc-multicluster-invocation.svg diff --git a/docs/user/images/nvcf-high-level-stack.svg b/docs/self-managed-1.0/images/nvcf-high-level-stack.svg similarity index 100% rename from docs/user/images/nvcf-high-level-stack.svg rename to docs/self-managed-1.0/images/nvcf-high-level-stack.svg diff --git a/docs/user/images/nvcf-http-invocation-path.svg b/docs/self-managed-1.0/images/nvcf-http-invocation-path.svg similarity index 100% rename from docs/user/images/nvcf-http-invocation-path.svg rename to docs/self-managed-1.0/images/nvcf-http-invocation-path.svg diff --git a/docs/user/images/nvcf-http-multicluster-invocation.svg b/docs/self-managed-1.0/images/nvcf-http-multicluster-invocation.svg similarity index 100% rename from docs/user/images/nvcf-http-multicluster-invocation.svg rename to docs/self-managed-1.0/images/nvcf-http-multicluster-invocation.svg diff --git a/docs/user/images/nvcf-llm-invocation-path.svg b/docs/self-managed-1.0/images/nvcf-llm-invocation-path.svg similarity index 100% rename from docs/user/images/nvcf-llm-invocation-path.svg rename to docs/self-managed-1.0/images/nvcf-llm-invocation-path.svg diff --git a/docs/user/images/nvcf-llm-multicluster-invocation.svg b/docs/self-managed-1.0/images/nvcf-llm-multicluster-invocation.svg similarity index 100% rename from docs/user/images/nvcf-llm-multicluster-invocation.svg rename to docs/self-managed-1.0/images/nvcf-llm-multicluster-invocation.svg diff --git a/docs/user/images/nvcf-lls-session.png b/docs/self-managed-1.0/images/nvcf-lls-session.png similarity index 100% rename from docs/user/images/nvcf-lls-session.png rename to docs/self-managed-1.0/images/nvcf-lls-session.png diff --git a/docs/user/images/nvcf-multi-region-multi-cluster.svg b/docs/self-managed-1.0/images/nvcf-multi-region-multi-cluster.svg similarity index 100% rename from docs/user/images/nvcf-multi-region-multi-cluster.svg rename to docs/self-managed-1.0/images/nvcf-multi-region-multi-cluster.svg diff --git a/docs/user/images/self-hosted-min-topology.png b/docs/self-managed-1.0/images/self-hosted-min-topology.png similarity index 100% rename from docs/user/images/self-hosted-min-topology.png rename to docs/self-managed-1.0/images/self-hosted-min-topology.png diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/index.md b/docs/self-managed-1.0/index.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/index.md rename to docs/self-managed-1.0/index.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/infrastructure-sizing.md b/docs/self-managed-1.0/infrastructure-sizing.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/infrastructure-sizing.md rename to docs/self-managed-1.0/infrastructure-sizing.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/installation.md b/docs/self-managed-1.0/installation.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/installation.md rename to docs/self-managed-1.0/installation.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/llm-function-enablement.md b/docs/self-managed-1.0/llm-function-enablement.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/llm-function-enablement.md rename to docs/self-managed-1.0/llm-function-enablement.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/llm-gateway.md b/docs/self-managed-1.0/llm-gateway.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/llm-gateway.md rename to docs/self-managed-1.0/llm-gateway.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/llm-request-router-load-balancing.md b/docs/self-managed-1.0/llm-request-router-load-balancing.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/llm-request-router-load-balancing.md rename to docs/self-managed-1.0/llm-request-router-load-balancing.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/lls-installation.md b/docs/self-managed-1.0/lls-installation.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/lls-installation.md rename to docs/self-managed-1.0/lls-installation.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development.md b/docs/self-managed-1.0/local-development.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development.md rename to docs/self-managed-1.0/local-development.md diff --git a/docs/user/local-development/multi-cluster-cli.md b/docs/self-managed-1.0/local-development/multi-cluster-cli.md similarity index 100% rename from docs/user/local-development/multi-cluster-cli.md rename to docs/self-managed-1.0/local-development/multi-cluster-cli.md diff --git a/docs/user/local-development/multi-cluster-helmfile.md b/docs/self-managed-1.0/local-development/multi-cluster-helmfile.md similarity index 100% rename from docs/user/local-development/multi-cluster-helmfile.md rename to docs/self-managed-1.0/local-development/multi-cluster-helmfile.md diff --git a/docs/user/local-development/single-cluster-cli.md b/docs/self-managed-1.0/local-development/single-cluster-cli.md similarity index 100% rename from docs/user/local-development/single-cluster-cli.md rename to docs/self-managed-1.0/local-development/single-cluster-cli.md diff --git a/docs/user/local-development/single-cluster-helmfile.md b/docs/self-managed-1.0/local-development/single-cluster-helmfile.md similarity index 100% rename from docs/user/local-development/single-cluster-helmfile.md rename to docs/self-managed-1.0/local-development/single-cluster-helmfile.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/manifest.md b/docs/self-managed-1.0/manifest.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/manifest.md rename to docs/self-managed-1.0/manifest.md diff --git a/docs/self-managed-1.0/metrics/cassandra/metrics.md b/docs/self-managed-1.0/metrics/cassandra/metrics.md new file mode 100644 index 0000000000..e2836faf38 --- /dev/null +++ b/docs/self-managed-1.0/metrics/cassandra/metrics.md @@ -0,0 +1,11 @@ +# C\* Metrics + +| Metric name | Metric type | Source | Description | Unit (where applicable) | Interesting Labels | Required Filters (where applicable) | +| ------------------------------------------------------------ | ----------- | ------------------------------------- | ---------------------------------------------------- | ----------------------- | ------------------------- | ----------------------------------- | +| cassandra_client_authentication_failures_total | Counter | TBD | Client auth failures | | host | | +| cassandra_client_request_latency_seconds | Histogram | TBD | Client requests types and histogram of latency | seconds | host, operation, quantile | | +| cassandra_client_request_timeouts_total | Counter | TBD | Client request timeouts by type | | host, operation | | +| cassandra_endpoint_connection_timeouts_total | Counter | TBD | C\* endpoint connection timeouts | | endpoint, host | | +| cassandra_jvm_gc_estimated_collection_duration_seconds_total | Gauge | TBD | Time spent in garbage collection | seconds | cassandra_cluster, host | | +| spring_data_repository_invocations_seconds_sum | Counter | nvcf-invocation-service:41337/metrics | Number of seconds for each C\* invocation, by method | Seconds | method, state | namespace="\" | +| spring_data_repository_invocations_seconds_count | Counter | nvcf-invocation-service:41337/metrics | Count of invocations for C\* by method | | method, state | namespace="\" | diff --git a/docs/self-managed-1.0/metrics/ess/metrics.md b/docs/self-managed-1.0/metrics/ess/metrics.md new file mode 100644 index 0000000000..0462cda467 --- /dev/null +++ b/docs/self-managed-1.0/metrics/ess/metrics.md @@ -0,0 +1,6 @@ +# ESS Metrics + +| Metric name | Metric type | Source | Description | Unit (where applicable) | Interesting Labels | Required Filters (where applicable) | +| ------------------------------------------ | ----------- | ------------------------------------- | -------------------------------------- | ----------------------- | ------------------------------------------------------------------- | ----------------------------------- | +| ess_templates_rendered_total | Counter | ess-api:9464/metrics | Total number of ESS templates rendered | | function_id, function_version_id, status | | +| http_client_request_duration_seconds_count | Gauge | nvcf-invocation-service:41337/metrics | HTTP Request Duration | seconds | http_request_method, http_response_status_code, http_route, service | server_address="ess.ngc.nvidia.com" | diff --git a/docs/self-managed-1.0/metrics/grpc-proxy/metrics.md b/docs/self-managed-1.0/metrics/grpc-proxy/metrics.md new file mode 100644 index 0000000000..adaef77c74 --- /dev/null +++ b/docs/self-managed-1.0/metrics/grpc-proxy/metrics.md @@ -0,0 +1,58 @@ +# gRPC Proxy Metrics + +The NVCF gRPC proxy emits Prometheus metrics on Service `grpc` (and the headless variant `grpc-headless`) in namespace `nvcf`, port `metrics` (10083), path `/metrics`. The exposition format is Prometheus text (via `prometheus/client_golang`). + +The metric families below cover client connection health, the NATS pipe to workers, and gRPC worker session attachment. + +| Metric name | Metric type | Source | Description | Unit (where applicable) | Interesting Labels | Required Filters (where applicable) | +| -------------------------------------------------------- | ----------- | ---------------- | ------------------------------------------------------------------------------------------------------ | ----------------------- | ------------------------------ | ---------------------------------------------------- | +| nvcf_grpc_proxy_service_active_connections_total | Gauge | grpc:10083/metrics | Active client TCP connections to the gRPC proxy | | | namespace="nvcf" | +| nvcf_grpc_proxy_service_active_http_requests_total | Gauge | grpc:10083/metrics | Active client HTTP requests in-flight | | | namespace="nvcf" | +| nvcf_grpc_proxy_service_nats_in_bytes | Gauge | grpc:10083/metrics | Bytes received from the NATS connection | bytes | | namespace="nvcf" | +| nvcf_grpc_proxy_service_nats_in_msgs | Gauge | grpc:10083/metrics | Messages received from the NATS connection | | | namespace="nvcf" | +| nvcf_grpc_proxy_service_nats_out_bytes | Gauge | grpc:10083/metrics | Bytes sent to the NATS connection | bytes | | namespace="nvcf" | +| nvcf_grpc_proxy_service_nats_out_msgs | Gauge | grpc:10083/metrics | Messages sent to the NATS connection | | | namespace="nvcf" | +| nvcf_grpc_proxy_service_nats_error_total | Counter | grpc:10083/metrics | Errors observed on the NATS connection | | | namespace="nvcf" | +| nvcf_grpc_proxy_service_nats_failure_total | Counter | grpc:10083/metrics | NATS failures classified by cause | | reason | namespace="nvcf" | +| nvcf_grpc_proxy_service_nats_disconnect_total | Counter | grpc:10083/metrics | NATS disconnect events | | | namespace="nvcf" | +| nvcf_grpc_proxy_service_nats_reconnect_total | Counter | grpc:10083/metrics | NATS reconnect attempts | | | namespace="nvcf" | +| nvcf_grpc_proxy_service_nats_reconnects | Gauge | grpc:10083/metrics | Current reconnect attempt count | | | namespace="nvcf" | +| nvcf_grpc_proxy_service_nats_lame_duck_total | Counter | grpc:10083/metrics | NATS lame-duck messages observed | | | namespace="nvcf" | +| nvcf_grpc_proxy_service_session_init_seconds_bucket | Histogram | grpc:10083/metrics | Time spent initializing a gRPC worker session. Fires only on gRPC worker attach to port 10086. | seconds | is_reconnect, le | namespace="nvcf" | +| http_server_request_duration_seconds_bucket | Histogram | grpc:10083/metrics | RED metric: HTTP server request duration on the gRPC proxy. | seconds | http_request_method, http_response_status_code, http_route, le | namespace="nvcf" | +| http_server_request_body_size_bytes_bucket | Histogram | grpc:10083/metrics | HTTP server request body size | bytes | http_request_method, http_route, le | namespace="nvcf" | +| http_server_response_body_size_bytes_bucket | Histogram | grpc:10083/metrics | HTTP server response body size | bytes | http_request_method, http_route, le | namespace="nvcf" | +| rpc_client_duration_milliseconds_bucket | Histogram | grpc:10083/metrics | OpenTelemetry gRPC client RPC duration (per-RPC outcome) | milliseconds | rpc_service, rpc_method, rpc_grpc_status_code, le | namespace="nvcf" | + +## Notes + +- `nvcf_grpc_proxy_service_session_init_seconds_bucket` is the SLI for gRPC inference function health. HTTP inference functions invoked through the regular HTTP invocation gateway bypass the gRPC proxy entirely and do not register session-init samples here. +- The `nvcf_grpc_proxy_service_nats_*` family is a useful proxy signal for "is the gRPC proxy -> NATS pipe healthy" (in/out bytes and message deltas) and "are NATS upstreams stable" (reconnect and error counters). +- Per-RPC outcomes (success vs. error per call) are covered by the OpenTelemetry `rpc_client_*` family; aggregate proxy-side errors are covered by `nvcf_grpc_proxy_service_nats_error_total`. +- The NATS failure `reason` values are `certificate_expired`, `tls_verification`, `tls`, `authentication`, `timeout`, `connection`, and `other`. +- Standard Go runtime metrics (`go_*`) and process metrics (`process_*`) are also exposed on the same endpoint and follow upstream conventions. + +## Alert queries + +Alert on disconnect churn independently from successful reconnects: + +```promql +sum(increase(nvcf_grpc_proxy_service_nats_disconnect_total[5m])) > 0 +``` + +Group NATS errors by their bounded reason label: + +```promql +sum by (reason) (increase(nvcf_grpc_proxy_service_nats_failure_total[5m])) > 0 +``` + +Use `reason="certificate_expired"` for certificate expiry alerts. Use +`reason=~"tls_verification|tls"` for other TLS failures and +`reason="authentication"` for NATS credential failures. + +## Reproducing locally + +```bash +kubectl port-forward -n nvcf svc/grpc 10083:10083 +curl http://127.0.0.1:10083/metrics | grep -E '^nvcf_grpc_proxy_' +``` diff --git a/docs/self-managed-1.0/metrics/init-container/metrics.md b/docs/self-managed-1.0/metrics/init-container/metrics.md new file mode 100644 index 0000000000..c3b6752525 --- /dev/null +++ b/docs/self-managed-1.0/metrics/init-container/metrics.md @@ -0,0 +1,7 @@ +# Init Container Metrics + +| Metric name | Metric type | Source | Description | Unit (where applicable) | Interesting Labels | Required Filters (where applicable) | +| ------------------------------------------- | ----------- | ------------------------------------------ | -------------------------------------------- | ----------------------- | ------------------------------------------------ | ----------------------------------- | +| kube_pod_container_status_restarts_total | Counter | prometheus-kube-state-metrics:8080/metrics | Total number of restarts for init containers | | function_id, function_version_id, namespace, pod | container="init" | +| kube_pod_container_status_terminated_reason | Gauge | prometheus-kube-state-metrics:8080/metrics | Reason an init container terminated | | namespace, pod, reason | container="init" | +| kube_pod_container_status_waiting_reason | Gauge | prometheus-kube-state-metrics:8080/metrics | Reason an init container is waiting to start | | namespace, pod, reason | container="init" | diff --git a/docs/self-managed-1.0/metrics/invocation-service/metrics.md b/docs/self-managed-1.0/metrics/invocation-service/metrics.md new file mode 100644 index 0000000000..99c4d22c20 --- /dev/null +++ b/docs/self-managed-1.0/metrics/invocation-service/metrics.md @@ -0,0 +1,12 @@ +# Invocation Service Metrics + +| Metric name | Metric type | Source | Description | Unit (where applicable) | Interesting Labels | Required Filters (where applicable) | +| ----------------------------------------- | ----------- | ------------------------------------- | ------------------------------------------------- | ----------------------- | ------------------ | ----------------------------------------------------------------------------------------------- | +| axum_http_requests_total | Counter | nvcf-invocation-service:41337/metrics | Inovcation Service http requests count | | status | exported_endpoint=~"^/v2/nvcf.\*\|^/health\$", namespace="astro-tenant-nvcf-invocation-service" | +| axum_http_requests_duration_seconds_count | Counter | nvcf-invocation-service:41337/metrics | Invocation Service http request counts | | status | exported_endpoint=~"^/v2/nvcf.\*\|^/health\$", namespace="astro-tenant-nvcf-invocation-service" | +| axum_http_requests_duration_seconds_sum | Counter | nvcf-invocation-service:41337/metrics | Invocation Service durations of http requests | seconds | status | exported_endpoint=~"^/v2/nvcf.\*\|^/health\$", namespace="astro-tenant-nvcf-invocation-service" | +| app_invocation_error | Counter | nvcf-invocation-service:41337/metrics | Invocation Service invocation errors | | http_status_code | namespace=""astro-tenant-nvcf-invocation-service | +| container_cpu_usage_seconds_total | Counter | nvcf-invocation-service:41337/metrics | Container cpu usage (used for uptime calculation) | | container | container="nvcf-invocation-service" | +| container_memory_usage_bytes | Gauge | nvcf-invocation-service:41337/metrics | Container memory usage | bytes | container | container="nvcf-invocation-service" | +| aws_requests_status | Gauge | AWS Cloud Watch | AWS request status | | aws_status_code | namespace="astro-tenant-nvcf-invocation-service", service="nvcf-invocation-service" | +| nats_jetstream_publish | Counter | Pushed from Synadia | NATS Stream creation count | | namespace | namespace="astro-tenant-nvcf-invocation-service" | diff --git a/docs/self-managed-1.0/metrics/llm-api-gateway/metrics.md b/docs/self-managed-1.0/metrics/llm-api-gateway/metrics.md new file mode 100644 index 0000000000..49dcb8f3e8 --- /dev/null +++ b/docs/self-managed-1.0/metrics/llm-api-gateway/metrics.md @@ -0,0 +1,42 @@ +# LLM API Gateway Metrics + +The LLM API Gateway serves Prometheus metrics from +`llm-api-gateway:9464/metrics` when +`llmApiGateway.metrics.enabled` is `true`. + +The self-managed stack maps `global.observability.metrics.enabled` to this chart +value. The gateway metrics use the `llm_api_gateway_` service prefix and must not +emit legacy service-prefixed metric names. + +## Label Boundaries + +Use bounded labels only. Do not add request IDs, session IDs, function IDs, +organization IDs, project IDs, raw URLs, raw prompts, authorization values, or +other unbounded request fields as metric labels. + +## Metrics + +| Metric name | Type | Source endpoint | Labels | Notes | +| --- | --- | --- | --- | --- | +| `llm_api_gateway_http_requests_total` | Counter | `llm-api-gateway:9464/metrics` | `method`, `route`, `status` | Total inbound HTTP requests. `route` is the templated route, not the raw URL path. | +| `llm_api_gateway_http_request_duration_seconds` | Histogram | `llm-api-gateway:9464/metrics` | `method`, `route`, `status` | Inbound HTTP request duration in seconds. | +| `llm_api_gateway_http_active_requests` | Gauge | `llm-api-gateway:9464/metrics` | `method`, `route` | Current in-flight inbound HTTP requests. | +| `llm_api_gateway_upstream_requests_total` | Counter | `llm-api-gateway:9464/metrics` | `upstream`, `result`, `status` | Total outbound upstream requests. `upstream` is a bounded service name such as `llm-request-router`. | +| `llm_api_gateway_upstream_request_duration_seconds` | Histogram | `llm-api-gateway:9464/metrics` | `upstream`, `result`, `status` | Outbound upstream request duration in seconds. | +| `llm_api_gateway_llm_tokens_total` | Counter | `llm-api-gateway:9464/metrics` | `endpoint`, `token_type`, `stream` | LLM token counts reported by upstream providers. `token_type` is a bounded enum such as `prompt`, `completion`, or `total`. | +| `llm_api_gateway_provider_time_seconds` | Histogram | `llm-api-gateway:9464/metrics` | `endpoint`, `phase`, `stream` | Provider-reported timing phases in seconds. | +| `llm_api_gateway_stream_first_token_seconds` | Histogram | `llm-api-gateway:9464/metrics` | `endpoint` | Time from stream request start to first token in seconds. | +| `llm_api_gateway_stream_duration_seconds` | Histogram | `llm-api-gateway:9464/metrics` | `endpoint`, `status` | Total stream duration in seconds. | +| `llm_api_gateway_pubsub_publish_failures_total` | Counter | `llm-api-gateway:9464/metrics` | None | Number of messages that failed to publish. | +| `llm_api_gateway_pubsub_consume_failures_total` | Counter | `llm-api-gateway:9464/metrics` | None | Number of messages that failed to consume. | +| `llm_api_gateway_pubsub_consume_duration_seconds` | Histogram | `llm-api-gateway:9464/metrics` | None | Time to consume a message in seconds. | +| `llm_api_gateway_rate_limit_event_replication_lag_seconds` | Histogram | `llm-api-gateway:9464/metrics` | None | Lag between rate limit event creation and processing in seconds. | +| `llm_api_gateway_rate_limit_events_received_total` | Counter | `llm-api-gateway:9464/metrics` | None | Number of rate limit events received from the sync transport. | +| `llm_api_gateway_rate_limit_events_dropped_total` | Counter | `llm-api-gateway:9464/metrics` | `reason` | Number of received rate limit events dropped. `reason` is a bounded enum such as `same_cluster`, `old_message`, or `remote_apply_disabled`. | +| `llm_api_gateway_rate_limit_events_applied_total` | Counter | `llm-api-gateway:9464/metrics` | None | Number of rate limit events applied to the local limiter. | +| `llm_api_gateway_rate_limit_events_failed_apply_total` | Counter | `llm-api-gateway:9464/metrics` | None | Number of rate limit events that failed to apply locally. | +| `llm_api_gateway_rate_limit_events_dry_run_would_apply_total` | Counter | `llm-api-gateway:9464/metrics` | None | Number of rate limit events that would apply when remote application is disabled. | +| `llm_api_gateway_rate_limit_synchronizer_publish_duration_seconds` | Histogram | `llm-api-gateway:9464/metrics` | None | Time to publish a rate limit event in seconds. | +| `llm_api_gateway_rate_limit_synchronizer_queue_wait_seconds` | Histogram | `llm-api-gateway:9464/metrics` | None | Time spent queueing a rate limit event in seconds. | +| `llm_api_gateway_rate_limit_synchronizer_queue_length` | Gauge | `llm-api-gateway:9464/metrics` | None | Current rate limit synchronizer queue length. | +| `llm_api_gateway_rate_limit_synchronizer_events_dropped_total` | Counter | `llm-api-gateway:9464/metrics` | `reason` | Number of rate limit events dropped before publishing. `reason` is a bounded enum such as `old_message`. | diff --git a/docs/self-managed-1.0/metrics/llm-function-invocation-path.md b/docs/self-managed-1.0/metrics/llm-function-invocation-path.md new file mode 100644 index 0000000000..9b0ebb328f --- /dev/null +++ b/docs/self-managed-1.0/metrics/llm-function-invocation-path.md @@ -0,0 +1,100 @@ +# LLM Function Invocation Metrics Report + +This report covers the metrics available on the LLM function invocation path: +the LLM API Gateway, the LLM Request Router, and the Stargate client sidecar in +LLM function pods. + +## Scrape points + +| Component | Endpoint | Service name | Metric prefix | +| --- | --- | --- | --- | +| LLM API Gateway | `llm-api-gateway:9464/metrics` | `llm-api-gateway` | `llm_api_gateway_` | +| Rate limit sync worker | `:9464/metrics` when deployed with `METRICS_PORT=9464` | `llm-api-gateway-rate-limit-sync-worker` | `llm_api_gateway_` | +| LLM Request Router | `llm-request-router:9090/metrics` | `stargate` | `stargate_` | +| Stargate client sidecar | `:9089/metrics` by default | `stargate-client` | `stargate_client_` | + +The request-router chart passes `--metrics-port` and uses Stargate's default +metric prefix and trace service name. + +## LLM API Gateway + +| Metric | Labels | +| --- | --- | +| `llm_api_gateway_http_requests_total` | `method`, `route`, `status` | +| `llm_api_gateway_http_request_duration_seconds` | `method`, `route`, `status` | +| `llm_api_gateway_http_active_requests` | `method`, `route` | +| `llm_api_gateway_upstream_requests_total` | `upstream`, `result`, `status` | +| `llm_api_gateway_upstream_request_duration_seconds` | `upstream`, `result`, `status` | +| `llm_api_gateway_llm_tokens_total` | `endpoint`, `token_type`, `stream` | +| `llm_api_gateway_provider_time_seconds` | `endpoint`, `phase`, `stream` | +| `llm_api_gateway_stream_first_token_seconds` | `endpoint` | +| `llm_api_gateway_stream_duration_seconds` | `endpoint`, `status` | +| `llm_api_gateway_pubsub_publish_failures_total` | None | +| `llm_api_gateway_pubsub_consume_failures_total` | None | +| `llm_api_gateway_pubsub_consume_duration_seconds` | None | +| `llm_api_gateway_rate_limit_event_replication_lag_seconds` | None | +| `llm_api_gateway_rate_limit_events_received_total` | None | +| `llm_api_gateway_rate_limit_events_dropped_total` | `reason` | +| `llm_api_gateway_rate_limit_events_applied_total` | None | +| `llm_api_gateway_rate_limit_events_failed_apply_total` | None | +| `llm_api_gateway_rate_limit_events_dry_run_would_apply_total` | None | +| `llm_api_gateway_rate_limit_synchronizer_publish_duration_seconds` | None | +| `llm_api_gateway_rate_limit_synchronizer_queue_wait_seconds` | None | +| `llm_api_gateway_rate_limit_synchronizer_queue_length` | None | +| `llm_api_gateway_rate_limit_synchronizer_events_dropped_total` | `reason` | + +The sync worker reuses the same telemetry package and emits the rate limit +synchronizer and Pub/Sub metrics under the worker service name. + +## LLM Request Router + +| Metric | Labels | +| --- | --- | +| `stargate_requests_total` | `routing_key`, `model`, `inference_server_id`, `status` | +| `stargate_proxy_attempts_total` | `routing_key`, `model`, `inference_server_id`, `result` | +| `stargate_proxy_retries_total` | `routing_key`, `model`, `reason` | +| `stargate_routing_selections_total` | `routing_key`, `model`, `algorithm`, `selection` | +| `stargate_routing_kv_free_token_fallback_selections_total` | `routing_key`, `model`, `algorithm` | +| `stargate_proxy_retry_exhausted_total` | `routing_key`, `model`, `reason` | +| `stargate_admission_rejections_total` | `routing_key`, `model`, `reason` | +| `stargate_quic_connection_evictions_total` | `inference_server_id`, `reason` | +| `stargate_quic_hot_path_reconnect_total` | `inference_server_id`, `result` | +| `stargate_proxy_replay_buffer_bytes` | `model` | +| `stargate_proxy_duration_seconds` | `routing_key`, `model`, `inference_server_id` | +| `stargate_routing_duration_seconds` | `routing_key`, `model` | +| `stargate_active_inference_servers` | `routing_key`, `model` | + +## Stargate Client Sidecar + +| Metric | Labels | +| --- | --- | +| `target_info` | `service_version`, `service_name`, `commit` | +| `stargate_client_requests_inflight` | `model` | +| `stargate_client_requests_state` | `model`, `state` | +| `stargate_client_requests_state_input_tokens` | `model`, `state` | +| `stargate_client_requests_total` | `model`, `routing_key`, `status` | +| `stargate_client_request_time_to_response_headers_seconds` | `model`, `routing_key` | +| `stargate_client_request_time_to_first_output_seconds` | `model`, `routing_key` | +| `stargate_client_request_time_to_first_token_seconds` | `model`, `routing_key` | +| `stargate_client_request_duration_seconds` | `model`, `routing_key`, `status` | +| `stargate_client_request_input_tokens_total` | `model`, `routing_key`, `status` | +| `stargate_client_request_output_tokens_total` | `model`, `routing_key`, `status` | +| `stargate_client_request_input_tokens` | `model`, `routing_key`, `status` | +| `stargate_client_request_output_tokens` | `model`, `routing_key`, `status` | +| `stargate_client_registration_stream_connected` | `router` | +| `stargate_client_reverse_tunnel_connected` | `router` | +| `stargate_client_model_input_tps` | `model` | +| `stargate_client_model_output_tps` | `model` | +| `stargate_client_model_max_input_tps` | `model` | +| `stargate_client_model_max_output_tps` | `model` | +| `stargate_client_model_queue_size` | `model` | +| `stargate_client_model_queued_input_tokens` | `model` | +| `stargate_client_model_kv_cache_capacity_tokens` | `model` | +| `stargate_client_model_kv_cache_used_tokens` | `model` | +| `stargate_client_model_kv_cache_free_tokens` | `model` | +| `stargate_client_model_advertised_status` | `router`, `model`, `status` | +| `stargate_client_retryable_responses_total` | `inference_server_id`, `reason`, `status` | +| `stargate_client_nonretryable_failures_total` | `inference_server_id`, `reason` | + +Keep request IDs, session IDs, function IDs, organization IDs, project IDs, +authorization values, raw prompts, and raw URLs out of metric labels. diff --git a/docs/self-managed-1.0/metrics/llm-request-router/metrics.md b/docs/self-managed-1.0/metrics/llm-request-router/metrics.md new file mode 100644 index 0000000000..14cbed75e8 --- /dev/null +++ b/docs/self-managed-1.0/metrics/llm-request-router/metrics.md @@ -0,0 +1,40 @@ +# LLM Request Router Metrics + +The LLM Request Router serves Prometheus metrics from +`llm-request-router:9090/metrics` when +`llmRequestRouter.metrics.enabled` is `true`. The standalone request-router +chart defaults this value to `false`. + +The self-managed stack maps +`addons.llm.requestRouter.metrics.enabled` to this chart value. It defaults to +`true` when the LLM add-on is enabled. The request-router chart passes +`--metrics-port` and uses Stargate's default `stargate_` metric prefix and +`stargate` trace service name. + +## Label Boundaries + +Use bounded labels only. Keep `routing_key`, `model`, `inference_server_id`, +`algorithm`, `selection`, `status`, `result`, and `reason` to bounded service +dimensions. Do not add request IDs, session IDs, function IDs, organization +IDs, project IDs, raw URLs, raw prompts, authorization values, or other +unbounded request fields as metric labels. + +## Metrics + +| Metric name | Type | Source endpoint | Labels | Notes | +| --- | --- | --- | --- | --- | +| `stargate_requests_total` | Counter | `llm-request-router:9090/metrics` | `routing_key`, `model`, `inference_server_id`, `status` | Total proxied requests by selected backend and status. | +| `stargate_proxy_attempts_total` | Counter | `llm-request-router:9090/metrics` | `routing_key`, `model`, `inference_server_id`, `result` | Upstream proxy attempts by selected backend and result. | +| `stargate_proxy_retries_total` | Counter | `llm-request-router:9090/metrics` | `routing_key`, `model`, `reason` | Total proxy retries by retry reason. | +| `stargate_routing_selections_total` | Counter | `llm-request-router:9090/metrics` | `routing_key`, `model`, `algorithm`, `selection` | Primary and ranked fallback cluster choices used for upstream attempts. | +| `stargate_routing_kv_free_token_fallback_selections_total` | Counter | `llm-request-router:9090/metrics` | `routing_key`, `model`, `algorithm` | Routes selected after a higher-ranked candidate failed the KV free-token check. | +| `stargate_proxy_retry_exhausted_total` | Counter | `llm-request-router:9090/metrics` | `routing_key`, `model`, `reason` | Total requests that exhausted retry options. | +| `stargate_admission_rejections_total` | Counter | `llm-request-router:9090/metrics` | `routing_key`, `model`, `reason` | Requests rejected by local input-work admission control. | +| `stargate_quic_connection_evictions_total` | Counter | `llm-request-router:9090/metrics` | `inference_server_id`, `reason` | Total QUIC pool evictions by backend and reason. | +| `stargate_quic_hot_path_reconnect_total` | Counter | `llm-request-router:9090/metrics` | `inference_server_id`, `result` | Direct QUIC reconnect attempts from the proxy hot path. | +| `stargate_tls_reloads_total` | Counter | `llm-request-router:9090/metrics` | `material_type`, `result` | Mounted TLS server identity reload attempts by result. | +| `stargate_tls_certificate_expiry_seconds` | Gauge | `llm-request-router:9090/metrics` | `material_type` | Unix timestamp when the active server identity expires. | +| `stargate_proxy_replay_buffer_bytes` | Histogram | `llm-request-router:9090/metrics` | `model` | Proxied request replay buffer size in bytes. | +| `stargate_proxy_duration_seconds` | Histogram | `llm-request-router:9090/metrics` | `routing_key`, `model`, `inference_server_id` | Time to first byte from upstream in seconds. | +| `stargate_routing_duration_seconds` | Histogram | `llm-request-router:9090/metrics` | `routing_key`, `model` | Load-balancer decision time in seconds. | +| `stargate_active_inference_servers` | Gauge | `llm-request-router:9090/metrics` | `routing_key`, `model` | Currently routable inference servers for a routing target. | diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/metrics-index.md b/docs/self-managed-1.0/metrics/metrics-index.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/metrics-index.md rename to docs/self-managed-1.0/metrics/metrics-index.md diff --git a/docs/self-managed-1.0/metrics/nvcf-api/metrics.md b/docs/self-managed-1.0/metrics/nvcf-api/metrics.md new file mode 100644 index 0000000000..3057b65d2a --- /dev/null +++ b/docs/self-managed-1.0/metrics/nvcf-api/metrics.md @@ -0,0 +1,6 @@ +# NVCF API Metrics + +| Metric name | Metric type | Source | Description | Unit (where applicable) | Interesting Labels | Required Filters (where applicable) | +| ------------------------------------------ | ----------- | ---------------------------------------- | ---------------------------------------------- | ----------------------- | ------------------------------------------------------------------ | ----------------------------------------------------------------------------------- | +| logback_events_total | Counter | nvcf-api: nvcf-api-service:10083/metrics | Log event level counts | | level | service="strap-api" | +| http_server_request_duration_seconds_count | Counter | nvcf-api: nvcf-api-service:10083/metrics | Response codes for various endpoints, nvcf api | | http_request_method, http_response_status_code, http_route, method | server_address="api.nvcf.nvidia.com", job="strap-api", namespace="gdn-strap-api-fp" | diff --git a/docs/self-managed-1.0/metrics/sis-spot/metrics.md b/docs/self-managed-1.0/metrics/sis-spot/metrics.md new file mode 100644 index 0000000000..b92d3eaada --- /dev/null +++ b/docs/self-managed-1.0/metrics/sis-spot/metrics.md @@ -0,0 +1,5 @@ +# SIS/SPOT Metrics + +| Metric name | Metric type | Source | Description | Unit (where applicable) | Interesting Labels | Required Filters (where applicable) | +| ------------------------------------------ | ----------- | ---------------------------------- | -------------------------------- | ----------------------- | ------------------------------------------------------------------- | ----------------------------------------------------------------- | +| http_client_request_duration_seconds_count | Counter | spot-instance-service:9464/metrics | http_response codes with timings | | http_request_method, http_response_status_code, http_route, service | server_address="spot.gdn.nvidia.com", namespace="gdn-spot-api-fp" | diff --git a/docs/self-managed-1.0/metrics/state-metrics/metrics.md b/docs/self-managed-1.0/metrics/state-metrics/metrics.md new file mode 100644 index 0000000000..60cae207d6 --- /dev/null +++ b/docs/self-managed-1.0/metrics/state-metrics/metrics.md @@ -0,0 +1,13 @@ +# State Metrics Service Metrics + +| Metric name | Metric type | Source | Description | Unit (where applicable) | Interesting Labels | Required Filters (where applicable) | +| ------------------------------------------ | ----------- | ------------------------------------------ | ------------------------------------------------------- | ----------------------- | --------------------------------------------------------------------------------------------------- | --------------------------------------------------- | +| container_cpu_usage_seconds_total | Counter | prometheus-kube-state-metrics:8080/metrics | Container cpu usage (used for uptime calculation) | | container , function_id, function_version_id, namespace | namespace="astro-tenant-nvcf-state-metrics-service" | +| state_metrics_stage_duration_seconds | Gauge | nvcf-state-metrics-service:9464/metrics | Duration of various stages in the state metrics service | seconds | stage | namespace="astro-tenant-nvcf-state-metrics-service" | +| http_client_request_duration_seconds_count | Counter | nvcf-state-metrics-service:9464/metrics | http_response codes with timings | | http_request_method, http_response_status_code, http_route, service | service="nvcf-state-metrics-service" | +| nvcf_function_info | Gauge | nvcf-state-metrics-service:9464/metrics | Various function information | | container_image, endpoint, function_id, function_version_id, helm_chart, name, secrets | | +| nvcf_function_status | Gauge | nvcf-state-metrics-service:9464/metrics | Various function status | | account_display_name, account_name, function_id, function_version_id, name, nca_id, status, version | | +| nvcf_function_instances_current | Gauge | nvcf-state-metrics-service:9464/metrics | Currently deployed functions | | function_version_id, nca_id, version | | +| nvcf_function_queue_depth | Gauge | nvcf-state-metrics-service:9464/metrics | Queue metrics for functions | | account_display_name, account_name, function_id, function_version_id, name, nca_id, version | | +| nvcf_function_instances_current | Gauge | nvcf-state-metrics-service:9464/metrics | Number of instances for a function | | account_display_name, account_name, function_id, function_version_id, name, nca_id, version | | +| function_request_latency_bucket | Histogram | nvcf-state-metrics-service:9464/metrics | Invocation latency histogram | | function_id, function_version_id, le | | diff --git a/docs/self-managed-1.0/metrics/utils-container/metrics.md b/docs/self-managed-1.0/metrics/utils-container/metrics.md new file mode 100644 index 0000000000..ac2634cc00 --- /dev/null +++ b/docs/self-managed-1.0/metrics/utils-container/metrics.md @@ -0,0 +1,8 @@ +# Utils Container Metrics + +| Metric name | Metric type | Source | Description | Unit (where applicable) | Interesting Labels | Required Filters (where applicable) | +| ------------------------------------------- | ----------- | ------------------------------------------- | ------------------------------ | ----------------------- | ------------------------------------------------ | ----------------------------------- | +| kube_pod_container_status_restarts_total | Counter | prometheus-kube-state-metrics:8080/metrics | Pod restart count | | function_id, function_version_id, namespace, pod | container="utils" | +| kube_pod_container_status_terminated_reason | Gauge | prometheus-kube-state-metrics:8080/metrics | Reason pod was terminated | | namespace, pod, reason | container="utils" | +| kube_pod_container_status_waiting_reason | Gauge | prometheus-kube-state-metrics:8080/metrics | Reason pod is waiting to start | | namespace, pod | container="utils" | +| nvcf_worker_service_response_total | Gauge | prometheus-agent-metrics-utils:8010/metrics | | | | | diff --git a/docs/self-managed-1.0/metrics/vault-openbao/metrics.md b/docs/self-managed-1.0/metrics/vault-openbao/metrics.md new file mode 100644 index 0000000000..0ced56d893 --- /dev/null +++ b/docs/self-managed-1.0/metrics/vault-openbao/metrics.md @@ -0,0 +1,5 @@ +# Vault (implemented as OpenBao) Metrics + +| Metric name | Metric type | Source | Description | Unit (where applicable) | Interesting Labels | Required Filters (where applicable) | +| ------------------------------------------------------------------------------------------------- | ----------- | ------ | ----------- | ----------------------- | ------------------ | ----------------------------------- | +| For open telemetry metrics available, see [https://openbao.org/docs/internals/telemetry/metrics/](https://openbao.org/docs/internals/telemetry/metrics/) | | | | | | | diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/nvcf-ui.md b/docs/self-managed-1.0/nvcf-ui.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/nvcf-ui.md rename to docs/self-managed-1.0/nvcf-ui.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/observability.md b/docs/self-managed-1.0/observability.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/observability.md rename to docs/self-managed-1.0/observability.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/optional-enhancements-gpu.md b/docs/self-managed-1.0/optional-enhancements-gpu.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/optional-enhancements-gpu.md rename to docs/self-managed-1.0/optional-enhancements-gpu.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/optional-enhancements-standalone.md b/docs/self-managed-1.0/optional-enhancements-standalone.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/optional-enhancements-standalone.md rename to docs/self-managed-1.0/optional-enhancements-standalone.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/optional-enhancements.md b/docs/self-managed-1.0/optional-enhancements.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/optional-enhancements.md rename to docs/self-managed-1.0/optional-enhancements.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/pod-disruption-budgets.md b/docs/self-managed-1.0/pod-disruption-budgets.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/pod-disruption-budgets.md rename to docs/self-managed-1.0/pod-disruption-budgets.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/quickstart.md b/docs/self-managed-1.0/quickstart.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/quickstart.md rename to docs/self-managed-1.0/quickstart.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/registry-allowlist.md b/docs/self-managed-1.0/registry-allowlist.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/registry-allowlist.md rename to docs/self-managed-1.0/registry-allowlist.md diff --git a/docs/self-managed-1.0/runbooks/caches.md b/docs/self-managed-1.0/runbooks/caches.md new file mode 100644 index 0000000000..de9d7e82d7 --- /dev/null +++ b/docs/self-managed-1.0/runbooks/caches.md @@ -0,0 +1,54 @@ +# Caches Runbook + +## UCC & DDCS + +Caching is a critical component of the rendering pipeline. However, being that these +services are caches, data loss is tolerated by the rendering pipeline. Offline pods +or missing data will result in reduced simulation performance, i.e. increased load times. + +Each DDCS replica/pod is a partition of the data set. Partial data loss in DDCS +will trigger re-computation for some or all of the data set. + +UCC replicas serve as a pull-through cache for USD assets and are populated on-demand +during scene load time. Each replica will contain a distinct +copy of the source content. Data loss of a pod means traffic serviced by that peer +may see reduced performance as the cache is rebuilt from requests to the source. + +### Cache Co-locality + +For maximum performance it is recommended that caches are deployed as close to the +pods they service as possible. We recommend placing one cache per availability zone +to reduce network transit times between the caches and the render nodes. It is also +recommended to keep all render nodes that are participating in a single simulation +in the same availability zone to further reduce network latency. + +## Kubernetes Pod Health Monitoring with Prometheus + +Monitoring Kubernetes with Prometheus is essential to detect signs of unhealthy +pods before they impact your application. Below are just a few Prometheus metrics +that can be monitored for issues. + +| Metric | Signal | +| --- | --- | +| `kube_pod_status_ready` | Shows if a pod is ready or not to accept traffic. Pods in non-ready states for extended periods of time (e.g. more than 5min) indicate an unresolved problem. | +| `kube_pod_container_status_restarts_total` | Counts total restarts per container. High or increasing values indicate crashes or instability. Cache pods are long running processes and do not restart unless they encounter fatal errors. | +| `kube_pod_container_status_waiting_reason` | Waiting reasons such as `CrashLoopBackOff`, indicate persistent failures and restarts in the cache applications. | +| `kube_pod_status_phase` | When pod status is not `Running` for an extended period, this indicates a persistent failure state. | +| `container_cpu_usage_seconds_total` | Although more difficult to monitor, cache pods that consume large amounts of CPU cycles without traffic may indicate compaction issues. | + +## Removing Unhealthy Pods and PVCs + +Cache pods maintain a persistent data set. When cache services are in an unhealthy +state, (due to crashes, restarts etc.) it is recommended to delete the pod(s) and +storage to attempt to return the service to normal operation. + +1. Delete the PVCs for the Pod *before* restarting it: + + `kubectl delete pvc -n ` + +2. Delete the Pod: + + `kubectl delete pod -n ` + +Monitor the offending pod(s) and ensure that Kubernetes correctly re-creates and +attaches new PVCs to recreated Cache pod(s). diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/control-plane-key-rotation-mek.md b/docs/self-managed-1.0/runbooks/control-plane-key-rotation-mek.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/control-plane-key-rotation-mek.md rename to docs/self-managed-1.0/runbooks/control-plane-key-rotation-mek.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/index.md b/docs/self-managed-1.0/runbooks/index.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/index.md rename to docs/self-managed-1.0/runbooks/index.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/overview.md b/docs/self-managed-1.0/runbooks/overview.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/overview.md rename to docs/self-managed-1.0/runbooks/overview.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/transport-tls-rotation.md b/docs/self-managed-1.0/runbooks/transport-tls-rotation.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/transport-tls-rotation.md rename to docs/self-managed-1.0/runbooks/transport-tls-rotation.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/update-nvcf-account-quota-limits.md b/docs/self-managed-1.0/runbooks/update-nvcf-account-quota-limits.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/update-nvcf-account-quota-limits.md rename to docs/self-managed-1.0/runbooks/update-nvcf-account-quota-limits.md diff --git a/docs/user/samples/configs/cp-env-eks-example.yaml b/docs/self-managed-1.0/samples/configs/cp-env-eks-example.yaml similarity index 100% rename from docs/user/samples/configs/cp-env-eks-example.yaml rename to docs/self-managed-1.0/samples/configs/cp-env-eks-example.yaml diff --git a/docs/user/samples/configs/cp-example-secrets.yaml b/docs/self-managed-1.0/samples/configs/cp-example-secrets.yaml similarity index 100% rename from docs/user/samples/configs/cp-example-secrets.yaml rename to docs/self-managed-1.0/samples/configs/cp-example-secrets.yaml diff --git a/docs/user/samples/configs/local-dev-env.yaml b/docs/self-managed-1.0/samples/configs/local-dev-env.yaml similarity index 100% rename from docs/user/samples/configs/local-dev-env.yaml rename to docs/self-managed-1.0/samples/configs/local-dev-env.yaml diff --git a/docs/user/samples/scripts/create-nvcr-pull-secrets.sh b/docs/self-managed-1.0/samples/scripts/create-nvcr-pull-secrets.sh similarity index 100% rename from docs/user/samples/scripts/create-nvcr-pull-secrets.sh rename to docs/self-managed-1.0/samples/scripts/create-nvcr-pull-secrets.sh diff --git a/docs/user/samples/scripts/force-cleanup-nvcf.sh b/docs/self-managed-1.0/samples/scripts/force-cleanup-nvcf.sh similarity index 100% rename from docs/user/samples/scripts/force-cleanup-nvcf.sh rename to docs/self-managed-1.0/samples/scripts/force-cleanup-nvcf.sh diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/streaming-functions.md b/docs/self-managed-1.0/streaming-functions.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/streaming-functions.md rename to docs/self-managed-1.0/streaming-functions.md diff --git a/docs/user/task-creation.md b/docs/self-managed-1.0/task-creation.md similarity index 100% rename from docs/user/task-creation.md rename to docs/self-managed-1.0/task-creation.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/third-party-registries.md b/docs/self-managed-1.0/third-party-registries.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/third-party-registries.md rename to docs/self-managed-1.0/third-party-registries.md diff --git a/docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/troubleshooting.md b/docs/self-managed-1.0/troubleshooting.md similarity index 100% rename from docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/troubleshooting.md rename to docs/self-managed-1.0/troubleshooting.md diff --git a/docs/user/autoscaling/architecture.md b/docs/self-managed/autoscaling/architecture.md similarity index 91% rename from docs/user/autoscaling/architecture.md rename to docs/self-managed/autoscaling/architecture.md index b4dc436a00..3aeee7950f 100644 --- a/docs/user/autoscaling/architecture.md +++ b/docs/self-managed/autoscaling/architecture.md @@ -57,7 +57,7 @@ instance and concurrency data. These metrics must reach the backend that the autoscaler queries. The backend can be bundled VictoriaMetrics or an existing PromQL-compatible -service. See [Observability Configuration](../observability.md) for backend, +service. See [Observability Configuration](/nvcf/observability/observability) for backend, endpoint, and authentication settings. The autoscaler reports `not ready` until the query endpoint responds. @@ -72,7 +72,7 @@ Coordination relies on Cassandra TTLs to recover from failures without operator ## See Also -- [Configure Autoscaling](../configure-autoscaling.md) for setting per-function scaling bounds, factors, thresholds, and stickiness via the NVCF API. +- [Configure Autoscaling](/nvcf/overview/configure-autoscaling) for setting per-function scaling bounds, factors, thresholds, and stickiness via the NVCF API. - [Function Autoscaler Operations](./operations.md) for health endpoints and common issues. -- [Function Autoscaler Observability](./observability.md) for emitted metrics, traces, and logs. +- [Autoscaler Metrics](/nvcf/observability/autoscaler) for emitted metrics, traces, and logs. - [Helmfile Installation](../helmfile-installation.md#observability-configuration) for deployment profiles and metrics stack settings. diff --git a/docs/user/autoscaling/index.md b/docs/self-managed/autoscaling/index.md similarity index 81% rename from docs/user/autoscaling/index.md rename to docs/self-managed/autoscaling/index.md index 1f2711284e..fadd8885e9 100644 --- a/docs/user/autoscaling/index.md +++ b/docs/self-managed/autoscaling/index.md @@ -29,7 +29,7 @@ profile. The `control` and `all` profiles install the Function Autoscaler. The State Metrics must be enabled for `control` and `all`. With the default component modes, the control-plane stack also installs the shared collector and -VictoriaMetrics. See [Observability Configuration](../observability.md) for +VictoriaMetrics. See [Observability Configuration](/nvcf/observability/observability) for profile and backend settings. ## Architecture Overview @@ -49,7 +49,7 @@ sequence and bucket model. ## See Also - [Architecture](./architecture.md) for components, data flow, and the Cassandra LWT lock behavior that elects the discovery leader. -- [Configure Autoscaling](../configure-autoscaling.md) for setting per-function scaling bounds, factors, thresholds, and stickiness via the NVCF API. +- [Configure Autoscaling](/nvcf/overview/configure-autoscaling) for setting per-function scaling bounds, factors, thresholds, and stickiness via the NVCF API. - [Function Autoscaler Operations](./operations.md) for health endpoints and operational guidance. -- [Function Autoscaler Observability](./observability.md) for the metrics, traces, and logs emitted by the service. -- [Observability Configuration](../observability.md) for profiles and metrics backend configuration. +- [Autoscaler Metrics](/nvcf/observability/autoscaler) for the metrics, traces, and logs emitted by the service. +- [Observability Configuration](/nvcf/observability/observability) for profiles and metrics backend configuration. diff --git a/docs/user/autoscaling/operations.md b/docs/self-managed/autoscaling/operations.md similarity index 85% rename from docs/user/autoscaling/operations.md rename to docs/self-managed/autoscaling/operations.md index b694585700..b247144c6e 100644 --- a/docs/user/autoscaling/operations.md +++ b/docs/self-managed/autoscaling/operations.md @@ -2,7 +2,7 @@ The self-managed stack deploys the Function Autoscaler for the `control` and `all` observability profiles. State Metrics must remain enabled for both. See -[Observability Configuration](../observability.md) for profile and metrics +[Observability Configuration](/nvcf/observability/observability) for profile and metrics backend settings. ## Verify the deployment @@ -61,7 +61,7 @@ failures change readiness instead. ## See also -- [Function Autoscaler Observability](./observability.md) for the metrics and traces referenced in the symptoms above. -- [Configure Autoscaling](../configure-autoscaling.md) for setting per-function scaling bounds and policy via the NVCF API. +- [Autoscaler Metrics](/nvcf/observability/autoscaler) for the metrics and traces referenced in the symptoms above. +- [Configure Autoscaling](/nvcf/overview/configure-autoscaling) for setting per-function scaling bounds and policy via the NVCF API. - [Architecture](./architecture.md) for the component layout these symptoms map to. -- [Observability Configuration](../observability.md) for shared stack settings. +- [Observability Configuration](/nvcf/observability/observability) for shared stack settings. diff --git a/docs/user/control-plane-installation.md b/docs/self-managed/control-plane-installation.md similarity index 53% rename from docs/user/control-plane-installation.md rename to docs/self-managed/control-plane-installation.md index b1b3ecb7f6..fbd1ccdce5 100644 --- a/docs/user/control-plane-installation.md +++ b/docs/self-managed/control-plane-installation.md @@ -3,13 +3,13 @@ # Control Plane Installation -For a local k3d fresh install, start with the [Quickstart](./quickstart.md). The quickstart uses `nvcf-cli self-hosted up` to install the control plane, register the local k3d cluster, install NVCA, and run basic health checks. +For a local k3d fresh install, start with the [Quickstart](/nvcf/overview/quickstart). The quickstart uses `nvcf-cli self-hosted up` to install the control plane, register the local k3d cluster, install NVCA, and run basic health checks. Use [Helmfile Installation](./helmfile-installation.md) when you need manual release control, partial recovery, or upgrade operations. ## Installation guides -- [Quickstart](./quickstart.md) - use `nvcf-cli self-hosted up` for local k3d one-click installation. +- [Quickstart](/nvcf/overview/quickstart) - use `nvcf-cli self-hosted up` for local k3d one-click installation. - [Helmfile Installation](./helmfile-installation.md) - use `helmfile` for manual control-plane deployment. diff --git a/docs/user/control-plane-operations.md b/docs/self-managed/control-plane-operations.md similarity index 97% rename from docs/user/control-plane-operations.md rename to docs/self-managed/control-plane-operations.md index a4e46ea950..a107a88370 100644 --- a/docs/user/control-plane-operations.md +++ b/docs/self-managed/control-plane-operations.md @@ -15,7 +15,7 @@ When a user stores a secret through the NVCF API, ESS encrypts it with the activ ### Key Rotation Runbooks -- [control-plane-runbook-mek-rotation](runbooks/control-plane-key-rotation-mek.md) — Rotate the master encryption key stored in OpenBAO. +- [control-plane-runbook-mek-rotation](./runbooks/control-plane-key-rotation-mek.md) — Rotate the master encryption key stored in OpenBAO. ## Basic Operations @@ -290,4 +290,4 @@ rollback. ## Observability -For observability configuration and reference architecture, see [self-hosted-observability](./observability.md). +For observability configuration and reference architecture, see [self-hosted-observability](/nvcf/observability/observability). diff --git a/docs/user/csp-end-to-end-example-installation.md b/docs/self-managed/csp-end-to-end-example-installation.md similarity index 99% rename from docs/user/csp-end-to-end-example-installation.md rename to docs/self-managed/csp-end-to-end-example-installation.md index 62b64e23f7..c09e945c3f 100644 --- a/docs/user/csp-end-to-end-example-installation.md +++ b/docs/self-managed/csp-end-to-end-example-installation.md @@ -15,7 +15,7 @@ names. Substitute the equivalents for GKE, AKS, or on-prem. For a deeper reference on each release and on values, see [Helmfile Installation](../helmfile-installation). For pulling and mirroring -the bundles and images, see [Image Mirroring](../image-mirroring). +the bundles and images, see [Image Mirroring](/nvcf/overview/image-mirroring). This guide assumes you have already downloaded and extracted the control-plane @@ -74,13 +74,13 @@ them. Each cluster needs: - A default-capable `StorageClass` with dynamic provisioning. On EKS this is `gp3`, backed by the EBS CSI driver. Substitute your provider's class name. - The compute (GPU) cluster needs a GPU operator (real or the fake GPU operator - for non-GPU validation). See [Fake GPU Operator](../fake-gpu-operator). + for non-GPU validation). See [Fake GPU Operator](/nvcf/compute-plane/fake-gpu-operator). - The compute cluster needs the [SMB CSI driver](https://github.com/kubernetes-csi/csi-driver-smb) (`smb.csi.k8s.io`). NVCA uses it for shared model cache storage that function worker pods mount. Install and verify the driver before registering the GPU cluster. See the - [Self-Managed Clusters prerequisites](../self-managed-clusters#prerequisites) + [Self-Managed Clusters prerequisites](/nvcf/compute-plane/self-managed-clusters#prerequisites) for the installation command. Both clusters must be reachable through `kubectl` contexts: diff --git a/docs/user/gateway-routing.md b/docs/self-managed/gateway-routing.md similarity index 98% rename from docs/user/gateway-routing.md rename to docs/self-managed/gateway-routing.md index ea96dc1e9f..45d322d2a5 100644 --- a/docs/user/gateway-routing.md +++ b/docs/self-managed/gateway-routing.md @@ -416,7 +416,7 @@ HTTPS. | Install path | Gateway values to use | | --- | --- | -| [Quickstart](./quickstart.md) | Do not use these remote Gateway values. The quickstart uses local k3d route hostnames. | +| [Quickstart](/nvcf/overview/quickstart) | Do not use these remote Gateway values. The quickstart uses local k3d route hostnames. | | [Helmfile Installation](./helmfile-installation.md) | Use `GATEWAY_ADDR` as `global.domain`, and set `ingress.gatewayApi.gateways` to the Gateway names, namespaces, and listener names from Gateway quickstart. | ## Configure the CLI for Gateway access @@ -631,9 +631,9 @@ When the LLM invocation route is enabled in self-managed deployments, send OpenA ### Invocation Path Diagrams For local and multi-cluster invocation-path diagrams, see -[Generic HTTP Function Invocation](./generic-http-function-invocation.md), -[gRPC Function Invocation](./grpc-function-invocation.md), and -[LLM Gateway](./llm-gateway.md). +[Generic HTTP Function Invocation](/nvcf/overview/generic-http-function-invocation), +[gRPC Function Invocation](/nvcf/overview/g-rpc-function-invocation), and +[LLM Gateway](/nvcf/overview/llm-gateway). ### Vanity Gateway (Optional) @@ -829,7 +829,7 @@ Without the correct `Host` header, the gateway cannot match the request to an HT The NVCA agent on a self-managed GPU cluster has the same requirement when it reaches the control plane through a load-balancer-fronted gateway. Configure its host-header overrides in the operator values, not the CLI config. See -[self-managed-clusters](./cluster-management/self-managed.md). +[self-managed-clusters](/nvcf/compute-plane/self-managed-clusters). @@ -863,7 +863,7 @@ api_host: "api.a1b2c3d4.us-west-2.elb.amazonaws.com" invoke_host: "invocation.a1b2c3d4.us-west-2.elb.amazonaws.com" ``` -See [cli-configuration](./cli.md) for complete CLI configuration documentation. +See [cli-configuration](/nvcf/overview/cli) for complete CLI configuration documentation. ## Production: DNS and HTTPS @@ -1114,6 +1114,6 @@ For gRPC connection problems: ## Related Documentation - [helmfile-installation](./helmfile-installation.md) - Helmfile values that consume Gateway quickstart outputs -- [cli-configuration](./cli.md) - CLI configuration including Host header settings +- [cli-configuration](/nvcf/overview/cli) - CLI configuration including Host header settings - [Kubernetes Gateway API](https://gateway-api.sigs.k8s.io/) - [Envoy Gateway](https://gateway.envoyproxy.io/) diff --git a/docs/user/grpc-invocation-enablement.md b/docs/self-managed/grpc-invocation-enablement.md similarity index 100% rename from docs/user/grpc-invocation-enablement.md rename to docs/self-managed/grpc-invocation-enablement.md diff --git a/docs/user/helmfile-installation.md b/docs/self-managed/helmfile-installation.md similarity index 97% rename from docs/user/helmfile-installation.md rename to docs/self-managed/helmfile-installation.md index 5295f7d42d..957cff65d0 100644 --- a/docs/user/helmfile-installation.md +++ b/docs/self-managed/helmfile-installation.md @@ -3,12 +3,12 @@ This section covers manual Helmfile installation of the NVCF control plane and GPU cluster components for self-hosted NVCF deployments. -For a fresh install, start with the [Quickstart](./quickstart.md). Use this Helmfile guide when you need explicit release control, partial recovery, upgrades, or direct access to Helmfile values. +For a fresh install, start with the [Quickstart](/nvcf/overview/quickstart). Use this Helmfile guide when you need explicit release control, partial recovery, upgrades, or direct access to Helmfile values. This guide assumes you have already downloaded and extracted the `nvcf-self-managed-stack` Helmfile bundle (see -[download-nvcf-self-managed-stack](./image-mirroring.md)). Control-plane +[download-nvcf-self-managed-stack](/nvcf/overview/image-mirroring)). Control-plane commands run from inside that directory unless otherwise noted. The directory contains the control-plane Helmfile definitions, environment templates, and sample configurations referenced throughout. @@ -82,8 +82,8 @@ Helmfile `1.3.0+` re-introduced sequential execution via the `--sequential-helmf - A kubernetes cluster (CSP agnostic or on-prem). - Gateway API ingress prepared as described in [Gateway quickstart](./gateway-routing.md#gateway-quickstart) if you are exposing NVCF through Gateway API -- Artifacts must be available in a registry that your Kubernetes cluster can access. This can be the `nvcf-onprem` registry for NVCF control plane service artifacts, but function containers and helm charts must be configured to a user-managed registry. See [self-hosted-artifact-manifest](./manifest.md) and [self-hosted-image-mirroring](./image-mirroring.md). -- The `nvcf-self-managed-stack` repository must be downloaded to your local machine (see [download-nvcf-self-managed-stack](./image-mirroring.md)). +- Artifacts must be available in a registry that your Kubernetes cluster can access. This can be the `nvcf-onprem` registry for NVCF control plane service artifacts, but function containers and helm charts must be configured to a user-managed registry. See [self-hosted-artifact-manifest](/nvcf/overview/manifest) and [self-hosted-image-mirroring](/nvcf/overview/image-mirroring). +- The `nvcf-self-managed-stack` repository must be downloaded to your local machine (see [download-nvcf-self-managed-stack](/nvcf/overview/image-mirroring)). @@ -201,7 +201,7 @@ Environment configuration files define how NVCF is deployed in your specific env Set `HELMFILE_ENV` to your environment name and copy the base configuration. The filename must match `HELMFILE_ENV` because Helmfile uses it to select the environment file. The template below shows the values to configure for Amazon -EKS ([cp-env-eks-example.yaml](https://raw.githubusercontent.com/NVIDIA/nvcf/main/docs/user/samples/configs/cp-env-eks-example.yaml)). +EKS ([cp-env-eks-example.yaml](https://raw.githubusercontent.com/NVIDIA/nvcf/main/docs/overview/samples/configs/cp-env-eks-example.yaml)). ```bash cd path/to/nvcf-self-managed-stack @@ -371,7 +371,7 @@ VictoriaMetrics, State Metrics, and the Function Autoscaler. Set the VictoriaMetrics storage class for the target cluster. To use a customer-managed backend or change component ownership, see -[Observability Configuration](./observability.md). For autoscaler health and +[Observability Configuration](/nvcf/observability/observability). For autoscaler health and backend checks, see [Function Autoscaler Operations](./autoscaling/operations.md). @@ -498,9 +498,9 @@ image: ``` -If you have mirrored NVCF artifacts to your own registry (e.g., ECR), update both `helm.sources` and `image` to point to your mirror. See [self-hosted-image-mirroring](./image-mirroring.md) for details on mirroring artifacts. +If you have mirrored NVCF artifacts to your own registry (e.g., ECR), update both `helm.sources` and `image` to point to your mirror. See [self-hosted-image-mirroring](/nvcf/overview/image-mirroring) for details on mirroring artifacts. -When upgrading to a new `nvcf-self-managed-stack` version, re-mirror all artifacts before running `helmfile sync`. Each stack release may introduce new or updated container images and Helm charts. If these are not present in your private registry, pods will fail with `ImagePullBackOff`. For split installs, mirror both core stack resources listed in the [self-hosted-artifact-manifest](./manifest.md). If you deploy shared observability as a standalone stack, mirror the observability stack resource as well. +When upgrading to a new `nvcf-self-managed-stack` version, re-mirror all artifacts before running `helmfile sync`. Each stack release may introduce new or updated container images and Helm charts. If these are not present in your private registry, pods will fail with `ImagePullBackOff`. For split installs, mirror both core stack resources listed in the [self-hosted-artifact-manifest](/nvcf/overview/manifest). If you deploy shared observability as a standalone stack, mirror the observability stack resource as well. @@ -635,7 +635,7 @@ These credentials will then be used for function deployments. Note that if the r Copy the secrets template using the same `HELMFILE_ENV` value from Step 2. The filename must match `HELMFILE_ENV` because Helmfile loads the corresponding secrets file. The example below shows the required structure -([example-secrets.yaml](https://raw.githubusercontent.com/NVIDIA/nvcf/main/docs/user/samples/configs/cp-example-secrets.yaml)). You must +([example-secrets.yaml](https://raw.githubusercontent.com/NVIDIA/nvcf/main/docs/overview/samples/configs/cp-example-secrets.yaml)). You must replace all instances of `REPLACE_WITH_BASE64_DOCKER_CREDENTIAL` with your actual base64-encoded registry credentials. @@ -791,7 +791,7 @@ registry. If your `image` registry is private and your cluster nodes do not have built-in credential helpers, you must create Kubernetes `docker-registry` secrets in each NVCF namespace and configure the helmfile to reference them. 1. Create the pull secret in each NVCF namespace - ([create-nvcr-pull-secrets.sh](https://raw.githubusercontent.com/NVIDIA/nvcf/main/docs/user/samples/scripts/create-nvcr-pull-secrets.sh)): + ([create-nvcr-pull-secrets.sh](https://raw.githubusercontent.com/NVIDIA/nvcf/main/docs/overview/samples/scripts/create-nvcr-pull-secrets.sh)): ```bash export NGC_API_KEY="" @@ -1425,7 +1425,7 @@ each cluster. ## Next Steps After the control plane and GPU clusters are installed, proceed to -[Self-Managed Clusters](./cluster-management/self-managed.md) for NVCA +[Self-Managed Clusters](/nvcf/compute-plane/self-managed-clusters) for NVCA operations and troubleshooting. ## Uninstalling diff --git a/docs/user/installation.md b/docs/self-managed/installation.md similarity index 72% rename from docs/user/installation.md rename to docs/self-managed/installation.md index 75ac01dc6e..aab091ae7f 100644 --- a/docs/user/installation.md +++ b/docs/self-managed/installation.md @@ -2,14 +2,14 @@ Self-hosted NVCF installation includes the core components required for NVCF inference. Optional components such as caching and low latency streaming support are also available. Vanity Gateway routing is available only in stack packages that include the Vanity Gateway addon. NVCF UI is available only in stack packages that include the NVCF UI addon. -For a local k3d fresh install, start with the [Quickstart](./quickstart.md). The quickstart uses `nvcf-cli self-hosted up` to install the control plane, register the local k3d cluster, install NVCA, and run basic health checks. +For a local k3d fresh install, start with the [Quickstart](/nvcf/overview/quickstart). The quickstart uses `nvcf-cli self-hosted up` to install the control plane, register the local k3d cluster, install NVCA, and run basic health checks. -For a full list of required artifacts, see [self-hosted-artifact-manifest](./manifest.md). +For a full list of required artifacts, see [self-hosted-artifact-manifest](/nvcf/overview/manifest). -![Self-hosted component overview](images/nvcf-high-level-stack.svg) +![Self-hosted component overview](../overview/images/nvcf-high-level-stack.svg) -Want to try NVCF locally first? See [Local Development](../local-development) to create a k3d cluster, then use the [Quickstart](./quickstart.md) local k3d flow. +Want to try NVCF locally first? See [Local Development](/nvcf/overview/local-development) to create a k3d cluster, then use the [Quickstart](/nvcf/overview/quickstart) local k3d flow. @@ -17,7 +17,7 @@ Want to try NVCF locally first? See [Local Development](../local-development) to | Path | Use when | Starting point | | --- | --- | --- | -| Local one-click CLI installation | You want the fastest local k3d install and cluster registration path. | [Quickstart](./quickstart.md) | +| Local one-click CLI installation | You want the fastest local k3d install and cluster registration path. | [Quickstart](/nvcf/overview/quickstart) | | Helmfile installation | You need manual release control, partial recovery, upgrades, or detailed Helmfile operations. | [Helmfile Installation](./helmfile-installation.md) | The control plane and GPU cluster can be the same Kubernetes cluster or separate clusters when you use Helmfile or the explicit CLI install primitives. The quickstart supports only a single local k3d cluster. For a complete Amazon EKS example of both topologies, see the [CSP End-to-End Example](./csp-end-to-end-example-installation.md). @@ -25,7 +25,7 @@ The control plane and GPU cluster can be the same Kubernetes cluster or separate For remote installs, prepare the Gateway API ingress path and CLI endpoint configuration before registering GPU clusters or running post-install CLI checks. See [Helmfile Installation](./helmfile-installation.md), -[Self-Managed Clusters](./cluster-management/self-managed.md), and +[Self-Managed Clusters](/nvcf/compute-plane/self-managed-clusters), and [Gateway Routing](./gateway-routing.md). ## Overview @@ -37,18 +37,18 @@ Every installation path follows the same high-level sequence: 2. Make NVCF artifacts available to your Kubernetes clusters. Pull them directly from NGC when the clusters have NGC access, or follow the - [image mirroring instructions](./image-mirroring.md) to copy them to a + [image mirroring instructions](/nvcf/overview/image-mirroring) to copy them to a registry that the clusters can access. 3. Create or select Kubernetes cluster targets. You need a cluster for the control plane and a GPU cluster for function workloads. These can be the same cluster or separate clusters. -4. Install the self-hosted control plane. Use the [Quickstart](./quickstart.md) for a local k3d install or [Helmfile Installation](./helmfile-installation.md) for manual Helmfile operations. +4. Install the self-hosted control plane. Use the [Quickstart](/nvcf/overview/quickstart) for a local k3d install or [Helmfile Installation](./helmfile-installation.md) for manual Helmfile operations. -5. Register a GPU cluster and install the NVIDIA Cluster Agent. The local quickstart performs this step for the local k3d cluster. For manual installation paths, see [Self-Managed Clusters](./cluster-management/self-managed.md). +5. Register a GPU cluster and install the NVIDIA Cluster Agent. The local quickstart performs this step for the local k3d cluster. For manual installation paths, see [Self-Managed Clusters](/nvcf/compute-plane/self-managed-clusters). 6. Install Low Latency Streaming if needed for streaming workloads. See [LLS Installation](./lls-installation.md). -7. Install optional enhancements, such as caches, low latency streaming, or Vanity Gateway routing, NVCF UI when your stack package includes that addon. See [Optional Enhancements](./optional-enhancements.md). +7. Install optional enhancements, such as caches, low latency streaming, or Vanity Gateway routing, NVCF UI when your stack package includes that addon. See [Simulation Caches](/nvcf/compute-plane/simulation-caches), [LLS Installation](./lls-installation.md), [Gateway Routing](./gateway-routing.md), and [NVCF UI](./nvcf-ui.md). ## Kubernetes Cluster Requirements @@ -75,7 +75,7 @@ See [NVIDIA GPU Operator documentation](https://docs.nvidia.com/datacenter/cloud Fake GPU Operator for development and testing: For environments without actual GPU hardware, install the fake GPU operator to simulate -GPU resources. See [fake-gpu-operator](../fake-gpu-operator) for full instructions. +GPU resources. See [fake-gpu-operator](/nvcf/compute-plane/fake-gpu-operator) for full instructions. #### SMB CSI Driver @@ -84,7 +84,7 @@ The [SMB CSI driver](https://github.com/kubernetes-csi/csi-driver-smb) (`smb.csi.k8s.io`) must be installed on every GPU cluster. NVCA uses the driver for shared model cache storage that function worker pods mount. Install and verify the driver before registering the GPU cluster. See the -[Self-Managed Clusters prerequisites](./cluster-management/self-managed.md#prerequisites) +[Self-Managed Clusters prerequisites](/nvcf/compute-plane/self-managed-clusters#prerequisites) for the installation command. #### Network Policies @@ -106,8 +106,8 @@ Some cloud providers have minimum PVC size requirements. For example, AWS EBS gp ### Cluster Sizing and Storage -See [infrastructure-sizing](./infrastructure-sizing.md) for node pool specifications, storage +See [infrastructure-sizing](/nvcf/overview/infrastructure-sizing) for node pool specifications, storage recommendations, and three recommended sizing tiers (Development, Minimal HA, and Production). -![Self-hosted minimum topology](images/self-hosted-min-topology.png) +![Self-hosted minimum topology](../overview/images/self-hosted-min-topology.png) diff --git a/docs/user/llm-function-enablement.md b/docs/self-managed/llm-function-enablement.md similarity index 99% rename from docs/user/llm-function-enablement.md rename to docs/self-managed/llm-function-enablement.md index 3f184391c8..fd19c0e4cb 100644 --- a/docs/user/llm-function-enablement.md +++ b/docs/self-managed/llm-function-enablement.md @@ -7,8 +7,8 @@ route, and configures worker pods to use the `pylon` sidecar for model-aware routing. For LLM function payload shape and invocation examples, see -[Function Creation](./function-creation.md) and -[LLM Gateway](./llm-gateway.md). +[Function Creation](/nvcf/overview/function-creation) and +[LLM Gateway](/nvcf/overview/llm-gateway). For request-router deployment, trusted headers, and rollout validation, see [LLM Request Router Load Balancing](./llm-request-router-load-balancing.md). diff --git a/docs/user/llm-request-router-load-balancing.md b/docs/self-managed/llm-request-router-load-balancing.md similarity index 99% rename from docs/user/llm-request-router-load-balancing.md rename to docs/self-managed/llm-request-router-load-balancing.md index c4a12a3dec..72ec67cf0b 100644 --- a/docs/user/llm-request-router-load-balancing.md +++ b/docs/self-managed/llm-request-router-load-balancing.md @@ -260,7 +260,7 @@ The chart exposes `llm-request-router:9090/metrics` when request-router metrics are enabled. The current chart passes `--metrics-port` and uses Stargate's default `stargate_` metric prefix. -See [LLM Request Router Metrics](./metrics/llm-request-router/metrics.md) for +See [LLM Request Router Metrics](/nvcf/observability/llm-request-router) for metric names, labels, and scrape configuration. ## Troubleshoot diff --git a/docs/user/lls-installation.md b/docs/self-managed/lls-installation.md similarity index 98% rename from docs/user/lls-installation.md rename to docs/self-managed/lls-installation.md index 123a5b3d8a..94bb3f87be 100644 --- a/docs/user/lls-installation.md +++ b/docs/self-managed/lls-installation.md @@ -25,7 +25,7 @@ Self-hosted mode uses your container registry for container images and customer- The following diagram illustrates the request and streaming flow for self-hosted NVCF LLS, showing how a browser application invokes a function and establishes direct streaming channels to worker nodes: -![LLS Request and Streaming Flow Architecture](images/nvcf-lls-session.png) +![LLS Request and Streaming Flow Architecture](../overview/images/nvcf-lls-session.png) The architecture demonstrates a nine-step process: @@ -43,7 +43,7 @@ The architecture demonstrates a nine-step process: The following diagram shows the detailed architecture of a self-hosted NVCF LLS deployment in AWS VPC with EKS: -![Self-Hosted LLS AWS Architecture](images/lls-self-hosted-arch-streaming-proxy.png) +![Self-Hosted LLS AWS Architecture](../overview/images/lls-self-hosted-arch-streaming-proxy.png) Key components and data flows: @@ -127,7 +127,7 @@ The following artifacts must be mirrored for LLS deployment: - Streaming application images (e.g., `usd-composer`) -For detailed instructions on pulling these artifacts from NGC and pushing to your registry, see [self-hosted-image-mirroring](./image-mirroring.md). The [LLS-specific artifacts](./image-mirroring.md) section lists exactly what you need. +For detailed instructions on pulling these artifacts from NGC and pushing to your registry, see [self-hosted-image-mirroring](/nvcf/overview/image-mirroring). The [LLS-specific artifacts](/nvcf/overview/image-mirroring) section lists exactly what you need. When mirroring to ECR, the repository path must match your Helm values configuration. Ensure your `registryName` in Step 4 includes the same repository path you used when mirroring the chart. diff --git a/docs/user/nvcf-ui.md b/docs/self-managed/nvcf-ui.md similarity index 100% rename from docs/user/nvcf-ui.md rename to docs/self-managed/nvcf-ui.md diff --git a/docs/user/pod-disruption-budgets.md b/docs/self-managed/pod-disruption-budgets.md similarity index 100% rename from docs/user/pod-disruption-budgets.md rename to docs/self-managed/pod-disruption-budgets.md diff --git a/docs/user/registry-allowlist.md b/docs/self-managed/registry-allowlist.md similarity index 100% rename from docs/user/registry-allowlist.md rename to docs/self-managed/registry-allowlist.md diff --git a/docs/user/runbooks/control-plane-key-rotation-mek.md b/docs/self-managed/runbooks/control-plane-key-rotation-mek.md similarity index 100% rename from docs/user/runbooks/control-plane-key-rotation-mek.md rename to docs/self-managed/runbooks/control-plane-key-rotation-mek.md diff --git a/docs/user/runbooks/overview.md b/docs/self-managed/runbooks/overview.md similarity index 100% rename from docs/user/runbooks/overview.md rename to docs/self-managed/runbooks/overview.md diff --git a/docs/user/runbooks/transport-tls-rotation.md b/docs/self-managed/runbooks/transport-tls-rotation.md similarity index 100% rename from docs/user/runbooks/transport-tls-rotation.md rename to docs/self-managed/runbooks/transport-tls-rotation.md diff --git a/docs/user/runbooks/update-nvcf-account-quota-limits.md b/docs/self-managed/runbooks/update-nvcf-account-quota-limits.md similarity index 100% rename from docs/user/runbooks/update-nvcf-account-quota-limits.md rename to docs/self-managed/runbooks/update-nvcf-account-quota-limits.md diff --git a/docs/user/third-party-registries.md b/docs/self-managed/third-party-registries.md similarity index 99% rename from docs/user/third-party-registries.md rename to docs/self-managed/third-party-registries.md index 4a11200c9b..5772ea36e7 100644 --- a/docs/user/third-party-registries.md +++ b/docs/self-managed/third-party-registries.md @@ -11,7 +11,7 @@ In NVCF, **third-party registries** refer to container registries used for hosti When a function is created or deployed, these credentials are used by different components: -1. **NVCF API** - Stores and manages registry credentials, validates that images exist during function creation. See [self-hosted-api](./api.md) for the full API specification. +1. **NVCF API** - Stores and manages registry credentials, validates that images exist during function creation. See [self-hosted-api](/nvcf/overview/api) for the full API specification. 2. **NVCA (Cluster Agent)** - Renders Helm charts or pod specs for container functions and handles deployment lifecycle. Generates image pull credentials based on the registry type. 3. **Worker init container** - Responsible for pulling the function container images during deployment. diff --git a/docs/user/troubleshooting.md b/docs/self-managed/troubleshooting.md similarity index 99% rename from docs/user/troubleshooting.md rename to docs/self-managed/troubleshooting.md index 0b59dabf9c..e59ce3ad3e 100644 --- a/docs/user/troubleshooting.md +++ b/docs/self-managed/troubleshooting.md @@ -757,7 +757,7 @@ allows UDP, the NIC NSG blocks it by default. done ``` -See [Cloud Provider Network Requirements](./streaming-functions.md#cloud-provider-network-requirements) +See [Cloud Provider Network Requirements](/nvcf/overview/streaming-functions#cloud-provider-network-requirements) for the full CSP networking checklist. ### gRPC Session Resumption Fails @@ -804,7 +804,7 @@ creates a new session and returns a fresh request ID. Update the client to handle gRPC NotFound by discarding the stored request ID and retrying without it. -See [Session Resumption](./grpc-function-invocation.md#session-resumption) for +See [Session Resumption](/nvcf/overview/g-rpc-function-invocation#session-resumption) for the full request ID lifecycle. ## Getting Help @@ -1090,7 +1090,7 @@ echo "==============================================" ``` -[force-cleanup-nvcf.sh](https://raw.githubusercontent.com/NVIDIA/nvcf/main/docs/user/samples/scripts/force-cleanup-nvcf.sh) +[force-cleanup-nvcf.sh](https://raw.githubusercontent.com/NVIDIA/nvcf/main/docs/overview/samples/scripts/force-cleanup-nvcf.sh) **Usage:** diff --git a/docs/user/index.md b/docs/user/index.md deleted file mode 100644 index e231da3e6a..0000000000 --- a/docs/user/index.md +++ /dev/null @@ -1,18 +0,0 @@ -# NVIDIA Cloud Functions - -![NVIDIA Cloud Functions banner](images/nvcf-banner.svg) - -This guide provides information for deploying and operating NVCF in self-managed environments. - -- [Quickstart](./quickstart.md) - : Install the control plane, register a GPU cluster, and validate the deployment with the one-click CLI flow. -- [Deployment](./installation.md) - : Compare the one-click and Helmfile installation paths. -- [GPU Cluster Setup](./cluster-management/index.md) - : Connect GPU clusters to the NVCF control plane. -- [Configuration](./optional-enhancements.md) - : Configure gateway routing, registries, and optional enhancements. -- [Using Cloud Functions](./api.md) - : Create and invoke functions using the NVCF API and CLI. -- [Managed (Legacy)](../ngc-managed/cluster-management/ngc-managed.md) - : Documentation for the legacy NGC-managed NVCF platform (BYOC). diff --git a/docs/user/optional-enhancements-gpu.md b/docs/user/optional-enhancements-gpu.md deleted file mode 100644 index ae577b1223..0000000000 --- a/docs/user/optional-enhancements-gpu.md +++ /dev/null @@ -1,3 +0,0 @@ -# Optional Enhancements (GPU) - -This page is superseded. See [optional-enhancements](./optional-enhancements.md). diff --git a/docs/user/optional-enhancements-standalone.md b/docs/user/optional-enhancements-standalone.md deleted file mode 100644 index 228db0de9a..0000000000 --- a/docs/user/optional-enhancements-standalone.md +++ /dev/null @@ -1,3 +0,0 @@ -# Optional Enhancements (Standalone) - -This page is superseded. See [optional-enhancements](./optional-enhancements.md). diff --git a/docs/user/optional-enhancements.md b/docs/user/optional-enhancements.md deleted file mode 100644 index b069364717..0000000000 --- a/docs/user/optional-enhancements.md +++ /dev/null @@ -1,62 +0,0 @@ -# Optional Enhancements - -NVCF supports several optional components that can enhance your deployment's -performance, routing, and GPU cluster capabilities. Each component has its own -installation and configuration guide. - -## Pod Disruption Budgets - -- [Pod Disruption Budgets](pod-disruption-budgets.md) - Protect stateful infrastructure and control-plane services from simultaneous voluntary evictions during node drains and cluster upgrades - -## Low-Latency Streaming - -- [LLS Installation](lls-installation.md) - Required for streaming Cloud Functions using WebRTC - -## LLM Functions - -- [LLM Function Enablement](./llm-function-enablement.md) - Required for `functionType: "LLM"` functions using the LLM invocation gateway - -## NVCF Caches - -- [container-cache](./cluster-management/container-cache.md) - Accelerates container image pulls by caching layers locally -- [gxcache](./cluster-management/gxcache.md) - Shader caching for simulation and rendering workloads - -## Vanity Gateway - -Vanity Gateway is an optional HTTP gateway service for deployments that need -customer-facing hostnames or path mappings in front of the standard NVCF API and -invocation routes. It is available only in stack packages that include the -Vanity Gateway addon. Older packages do not contain the `vanity-gateway` release -or route values. - -When the addon is present and enabled, it is deployed as the `vanity-gateway` -service and exposed through the Gateway API route `vanity.` by default. -Enable it only when you need a vanity routing layer. Standard API, API Keys, -invocation, LLM invocation, and gRPC routes do not require it. See -[Gateway Routing](./gateway-routing.md#vanity-gateway-optional) for routing and -verification details. - -## NVCF UI - -NVCF UI is an optional addon that deploys a customer-facing NVCF admin-panel UI. -It is available only in stack packages that include the -NVCF UI addon. Older packages do not contain the `nvcf-ui` release -or route values. - -When the addon is present and enabled, it is deployed as the `nvcf-ui` -service and exposed through the Gateway API route `nvcf-ui.` by default. -See [Enabling NVCF UI](./nvcf-ui.md) for the enablement steps and -[Gateway Routing](./gateway-routing.md#nvcf-ui-optional) for routing and -verification details. - -## Physical Simulation Caches - -For an overview refer to [self-hosted-caches](./caches.md) - -- [Derived Data Cache Service](https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/) - Derived Data Cache Service -- [USD Content Cache](https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/) - USD Content Cache - - -These enhancements are supported for single-cluster Control Plane and GPU-only (BYOC) clusters. - - diff --git a/docs/user/release-notes/index.md b/docs/user/release-notes/index.md deleted file mode 100644 index 97799648b8..0000000000 --- a/docs/user/release-notes/index.md +++ /dev/null @@ -1,6 +0,0 @@ -# Release Notes - -Changes, fixes, and upgrade procedures for each self-managed NVCF release. - -Release notes and upgrade guides are published with their corresponding -versioned documentation. Select a release from the documentation version menu. diff --git a/docs/user/runbooks/index.md b/docs/user/runbooks/index.md deleted file mode 100644 index 41f0567ea4..0000000000 --- a/docs/user/runbooks/index.md +++ /dev/null @@ -1,12 +0,0 @@ ---- ---- - -# Runbooks - -A service runbook is a concise, structured document that outlines the standard operating procedures for running, troubleshooting, and maintaining various services. - -Its purpose is to give engineers (especially on-call or incident responders) the exact steps needed to quickly understand the service, handle common problems, and recover from failures without having to dig through large amounts of documentation. - -- [Control Plane Key Rotation](./control-plane-key-rotation-mek.md) -- [Transport TLS Rotation](./transport-tls-rotation.md) -- [Update NVCF Account Quota Limits](./update-nvcf-account-quota-limits.md) diff --git a/docs/version-catalog/compute-plane-1.0.yaml b/docs/version-catalog/compute-plane-1.0.yaml new file mode 100644 index 0000000000..fc6ac24fb3 --- /dev/null +++ b/docs/version-catalog/compute-plane-1.0.yaml @@ -0,0 +1,1667 @@ +version: 1 +target: main +registries: + public-helm: + host: https://helm.ngc.nvidia.com + namespace: nvidia/nvcf + repository_alias: nvcf + public-images: + host: nvcr.io + namespace: nvidia/nvcf + public-resources: + host: nvcr.io + namespace: nvidia/nvcf +publications: + - name: cert-manager-cainjector + type: image + version: v1.20.2 + registry: public-images + - name: cert-manager-controller + type: image + version: v1.20.2 + registry: public-images + - name: cert-manager-startupapicheck + type: image + version: v1.20.2 + registry: public-images + - name: cert-manager-webhook + type: image + version: v1.20.2 + registry: public-images + - name: oss-vault-k8s + type: image + version: 1.7.4 + registry: public-images + - name: nats-box + type: image + version: 0.19.7-nonroot + registry: public-images + - name: helm-nvcf-cert-manager + type: chart + version: 0.1.0 + registry: public-helm + chart_format: http + - name: helm-nvcf-pki + type: chart + version: 0.1.0 + registry: public-helm + chart_format: http + - name: helm-nvcf-ui + type: chart + version: 1.1.2 + registry: public-helm + chart_format: http + - name: nvcf-container-cache + type: image + version: v1.1.36 + registry: public-images + - name: nvcf-proxy-tls-certs + type: image + version: v1.2.10 + registry: public-images + - name: nvcf-container-cache + type: chart + version: 0.25.22 + registry: public-helm + chart_format: http + - name: nvcf-example-dashboards + type: chart + version: 1.6.0 + registry: public-helm + chart_format: http + - name: nvcf-observability-reference-stack + type: chart + version: 1.10.0 + registry: public-helm + chart_format: http + - name: admin-token-issuer-proxy + type: image + version: 1.1.2 + registry: public-images + - name: cassandra + type: image + version: 5.0.9-nv-2.0.5 + registry: public-images + - name: ess-agent + type: image + version: 1.4.1 + registry: public-images + - name: nats-server + type: image + version: 2.14.6-alpine3.22 + registry: public-images + - name: nvcf-api-keys-service + type: image + version: 1.9.1 + registry: public-images + - name: nvcf-cassandra-migrations + type: image + version: 0.17.6 + registry: public-images + - name: nvcf-image-credential-helper + type: image + version: 0.11.1 + registry: public-images + - name: nvcf-invocation-service + type: image + version: 0.12.1 + registry: public-images + - name: nvcf-nats-auth-callout-service + type: image + version: 0.8.3 + registry: public-images + - name: nvcf-openbao + type: image + version: 2.6.2-nv-1.3.4 + registry: public-images + - name: nvcf-openbao-migrations + type: image + version: 0.19.5 + registry: public-images + - name: nvcf-service-oss + type: image + version: 1.18.0 + registry: public-images + - name: nvcf-ui + type: image + version: 1.1.2 + registry: public-images + - name: nvcf-worker-init-oss + type: image + version: 1.2.1 + registry: public-images + - name: nvcf-worker-llm-credentials-oss + type: image + version: 1.1.2 + registry: public-images + - name: nvcf-worker-utils-oss + type: image + version: 1.2.3 + registry: public-images + - name: nvct-service-oss + type: image + version: 1.66.0 + registry: public-images + - name: reval-server + type: image + version: 0.20.2 + registry: public-images + - name: stargate + type: image + version: 0.18.0 + registry: public-images + - name: pylon + type: image + version: 0.18.0 + registry: public-images + - name: icms-service-oss + type: image + version: 0.7.2 + registry: public-images + - name: nvca-operator + type: image + version: 3.10.0 + registry: public-images + - name: nvca + type: image + version: 3.10.0 + registry: public-images + - name: llm-api-gateway + type: image + version: 0.14.2 + registry: public-images + - name: nvcf-ai-api-gateway-service + type: image + version: 1.35.1 + registry: public-images + - name: nvcf-ess + type: image + version: 0.5.0 + registry: public-images + - name: nvcf-function-autoscaler + type: image + version: 1.21.8 + registry: public-images + - name: nvcf-grpc-proxy + type: image + version: 1.33.5 + registry: public-images + - name: nvcf-notary + type: image + version: 1.14.1 + registry: public-images + - name: nvcf-ratelimiter + type: image + version: 1.17.3 + registry: public-images + - name: helm-admin-token-issuer-proxy + type: chart + version: 1.5.3 + registry: public-helm + chart_format: http + - name: helm-nvca-operator + type: chart + version: 1.28.0 + registry: public-helm + chart_format: http + - name: helm-nvcf-api + type: chart + version: 1.27.1 + registry: public-helm + chart_format: http + - name: helm-nvcf-api-keys + type: chart + version: 1.8.0 + registry: public-helm + chart_format: http + - name: helm-nvcf-cassandra + type: chart + version: 0.21.3 + registry: public-helm + chart_format: http + - name: helm-nvcf-ess-api + type: chart + version: 1.8.2 + registry: public-helm + chart_format: http + - name: helm-nvcf-function-autoscaler + type: chart + version: 0.5.2 + registry: public-helm + chart_format: http + - name: helm-nvcf-invocation-service + type: chart + version: 1.6.1 + registry: public-helm + chart_format: http + - name: helm-nvcf-llm-api-gateway + type: chart + version: 1.4.3 + registry: public-helm + chart_format: http + - name: helm-nvcf-nats + type: chart + version: 0.8.4 + registry: public-helm + chart_format: http + - name: helm-nvcf-nats-auth-callout-service + type: chart + version: 1.2.1 + registry: public-helm + chart_format: http + - name: helm-nvcf-notary-service + type: chart + version: 1.6.0 + registry: public-helm + chart_format: http + - name: helm-nvcf-nvct-api + type: chart + version: 1.6.0 + registry: public-helm + chart_format: http + - name: helm-nvcf-openbao-server + type: chart + version: 0.32.6 + registry: public-helm + chart_format: http + - name: helm-nvcf-rate-limiter + type: chart + version: 1.2.1 + registry: public-helm + chart_format: http + - name: helm-nvcf-sis + type: chart + version: 2.4.0 + registry: public-helm + chart_format: http + - name: helm-nvcf-vanity-gateway + type: chart + version: 0.5.0 + registry: public-helm + chart_format: http + - name: helm-reval + type: chart + version: 1.4.1 + registry: public-helm + chart_format: http + - name: helm-nvcf-grpc-proxy + type: chart + version: 1.7.4 + registry: public-helm + chart_format: http + - name: helm-nvcf-llm-request-router + type: chart + version: 1.14.1 + registry: public-helm + chart_format: http + - name: nvcf-cluster-topology + type: chart + version: 0.1.0 + registry: public-helm + chart_format: http + - name: nvcf-default-monitors + type: chart + version: 0.2.0 + registry: public-helm + chart_format: http + - name: nvcf-gateway-routes + type: chart + version: 1.18.2 + registry: public-helm + chart_format: http + - name: nvcf-otel-collector + type: chart + version: 0.2.0 + registry: public-helm + chart_format: http + - name: nvcf-cli + type: resource + version: 1.16.2 + registry: public-resources +version_overrides: + - name: nvcf-cli + type: resource + version: 1.16.2 + source: independent source release; not pinned by stack +publication_pending: + - helm-nvcf-state-metrics + - nvcf-compute-plane-stack + - nvcf-observability-stack + - nvcf-self-managed-stack + - nvcf-state-metrics-service +manifest: + entries: + - artifact_id: helm-nvcf-api-keys + plane: control + kind: chart + requirement: required + description: Deploys the API key management service. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/api-keys-colocated + - artifact_id: helm-nvcf-api + plane: control + kind: chart + requirement: required + description: Deploys the NVCF API service. + - artifact_id: helm-nvcf-cassandra + plane: control + kind: chart + requirement: required + description: Deploys Cassandra and its initialization jobs. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/cassandra + upstream_url: https://github.com/bitnami/charts/tree/main/bitnami/cassandra + - artifact_id: helm-nvcf-cert-manager + plane: control + kind: chart + requirement: required + description: Deploys the NVCF cert-manager configuration. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/cert-manager + upstream_url: https://github.com/cert-manager/cert-manager + - artifact_id: helm-nvcf-ess-api + plane: control + kind: chart + requirement: required + description: Deploys the Encrypted Secrets Service API. + - artifact_id: helm-nvcf-function-autoscaler + plane: control + kind: chart + requirement: required + description: Deploys the function autoscaler for observability-driven scaling. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/function-autoscaler + - artifact_id: helm-nvcf-grpc-proxy + plane: control + kind: chart + requirement: required + description: Deploys the gRPC proxy service. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/grpc-proxy + - artifact_id: helm-nvcf-invocation-service + plane: control + kind: chart + requirement: required + description: Deploys the HTTP invocation service. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/http-invocation + - artifact_id: helm-nvcf-nats-auth-callout-service + plane: control + kind: chart + requirement: required + description: Deploys the NATS authorization callout service. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nats-auth-callout + - artifact_id: helm-nvcf-nats + plane: control + kind: chart + requirement: required + description: Deploys NATS messaging for the control plane. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nats + upstream_url: https://github.com/nats-io/k8s + - artifact_id: helm-nvcf-notary-service + plane: control + kind: chart + requirement: required + description: Deploys the notary service for signing and validation. + - artifact_id: helm-nvcf-nvct-api + plane: control + kind: chart + requirement: required + description: Deploys the NVCF tenant API service. + - artifact_id: helm-nvcf-openbao-server + plane: control + kind: chart + requirement: required + description: Deploys OpenBao secret management. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/openbao + upstream_url: https://github.com/openbao/openbao-helm + - artifact_id: helm-nvcf-sis + plane: control + kind: chart + requirement: required + description: Deploys the Spot Instance Service. + - artifact_id: helm-reval + plane: control + kind: chart + requirement: required + description: Deploys the function revalidation service. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/helm-reval + - artifact_id: helm-admin-token-issuer-proxy + plane: control + kind: chart + requirement: required + description: Deploys the admin token issuer proxy. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/admin-token-issuer-proxy + - artifact_id: helm-nvcf-llm-api-gateway + plane: control + kind: chart + requirement: optional + description: Deploys the OpenAI-compatible LLM API gateway. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/llm-api-gateway + - artifact_id: helm-nvcf-llm-request-router + plane: control + kind: chart + requirement: optional + description: Deploys the LLM request router. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/llm-request-router + - artifact_id: helm-nvcf-pki + plane: control + kind: chart + requirement: optional + description: Provisions the OpenBao-backed ClusterIssuer for NVCF service TLS. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nvcf-pki + - artifact_id: helm-nvcf-rate-limiter + plane: control + kind: chart + requirement: required + description: Deploys request rate limiting for supported invocation paths. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/ratelimiter + - artifact_id: helm-nvcf-state-metrics + plane: control + kind: chart + requirement: required + description: Deploys NVCF state metrics for observability. + - artifact_id: helm-nvcf-ui + plane: control + kind: chart + requirement: optional + description: Deploys the optional NVCF UI admin panel. + - artifact_id: helm-nvcf-vanity-gateway + plane: control + kind: chart + requirement: optional + description: Deploys the optional vanity hostname gateway. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/vanity-gateway + - artifact_id: nvcf-example-dashboards + plane: control + kind: chart + requirement: optional + description: Deploys example Grafana dashboards for NVCF telemetry. + - artifact_id: nvcf-gateway-routes + plane: control + kind: chart + requirement: required + description: Deploys Gateway API routes for NVCF services. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/gateway-routes + - artifact_id: nvcf-default-monitors + plane: control + kind: chart + requirement: required + description: Deploys the default service and pod monitors for NVCF. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/observability/charts/nvcf-default-monitors + - artifact_id: nvcf-otel-collector + plane: control + kind: chart + requirement: required + description: Configures the OpenTelemetry Collector used by NVCF. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/observability/charts/nvcf-otel-collector + - artifact_id: opentelemetry-operator-chart + plane: control + kind: chart + requirement: required + description: Deploys the OpenTelemetry Operator. + upstream_url: https://github.com/open-telemetry/opentelemetry-operator + - artifact_id: prometheus-operator-crds + plane: control + kind: chart + requirement: required + description: Installs the Prometheus Operator custom resource definitions. + upstream_url: https://github.com/prometheus-community/helm-charts + - artifact_id: victoria-metrics-single + plane: control + kind: chart + requirement: required + description: Deploys the default metrics storage backend. + upstream_url: https://github.com/VictoriaMetrics/helm-charts + - artifact_id: nvcf-observability-reference-stack + plane: control + kind: chart + requirement: optional + description: Deploys a reference observability backend for evaluation. + - artifact_id: k8s + plane: control + kind: service-image + requirement: required + description: Provides Kubernetes command-line utilities for deployment jobs. + github_url: https://github.com/alpine-docker/k8s + - artifact_id: cert-manager-acmesolver + plane: control + kind: service-image + requirement: optional + description: Serves temporary ACME HTTP-01 domain-validation challenges. + upstream_url: https://github.com/cert-manager/cert-manager + - artifact_id: cert-manager-cainjector + plane: control + kind: service-image + requirement: required + description: Injects certificate authority data into Kubernetes resources. + upstream_url: https://github.com/cert-manager/cert-manager + - artifact_id: cert-manager-controller + plane: control + kind: service-image + requirement: required + description: Reconciles certificates and issuers for the control plane. + upstream_url: https://github.com/cert-manager/cert-manager + - artifact_id: cert-manager-startupapicheck + plane: control + kind: service-image + requirement: required + description: Verifies that the cert-manager API is ready. + upstream_url: https://github.com/cert-manager/cert-manager + - artifact_id: cert-manager-webhook + plane: control + kind: service-image + requirement: required + description: Validates and converts cert-manager API resources. + upstream_url: https://github.com/cert-manager/cert-manager + - artifact_id: nvcf-ess + plane: control + kind: service-image + requirement: required + description: Provides encrypted application secrets to NVCF workloads. + - artifact_id: nats-box + plane: control + kind: service-image + requirement: required + description: Provides NATS administration and diagnostic utilities. + upstream_url: https://github.com/nats-io/nats-box + - artifact_id: nats-server + plane: control + kind: service-image + requirement: required + description: Provides messaging for function deployment and invocation. + upstream_url: https://github.com/nats-io/nats-server + - artifact_id: nats-server-config-reloader + plane: shared + kind: service-image + requirement: required + description: Reloads NATS configuration for the control plane and optional NVIDIA Dynamo deployment. + upstream_url: https://github.com/nats-io/k8s + - artifact_id: nvcf-notary + plane: control + kind: service-image + requirement: required + description: Signs and validates functions and cluster nodes. + - artifact_id: nvcf-ai-api-gateway-service + plane: control + kind: service-image + requirement: optional + description: Serves the optional vanity hostname gateway. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/vanity-gateway + - artifact_id: nvcf-api-keys-service + plane: control + kind: service-image + requirement: required + description: Creates and manages NVCF API keys. + - artifact_id: icms-service-oss + plane: control + kind: service-image + requirement: required + description: Manages instance and cluster lifecycle operations. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/instance-cluster-management + - artifact_id: nvcf-grpc-proxy + plane: control + kind: service-image + requirement: required + description: Proxies bidirectional gRPC traffic between the control and compute planes. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/grpc-proxy + - artifact_id: nvcf-invocation-service + plane: control + kind: service-image + requirement: required + description: Routes stateless HTTP function invocation requests. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/http-invocation + - artifact_id: nvcf-nats-auth-callout-service + plane: control + kind: service-image + requirement: required + description: Authorizes NATS clients for NVCF services and workloads. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/nats-auth-callout + - artifact_id: nvcf-openbao + plane: control + kind: service-image + requirement: required + description: Stores and manages control-plane secrets. + upstream_url: https://github.com/openbao/openbao + - artifact_id: nvcf-openbao-migrations + plane: control + kind: service-image + requirement: required + description: Applies the OpenBao configuration required by NVCF. + github_url: https://github.com/NVIDIA/nvcf/tree/main/migrations/openbao + - artifact_id: nvcf-ratelimiter + plane: control + kind: service-image + requirement: required + description: Enforces request rate limits for supported invocation paths. + - artifact_id: nvcf-service-oss + plane: control + kind: service-image + requirement: required + description: Provides the primary NVCF control-plane API. + - artifact_id: nvct-service-oss + plane: control + kind: service-image + requirement: required + description: Provides tenant-scoped NVCF control-plane operations. + - artifact_id: oss-vault-k8s + plane: control + kind: service-image + requirement: required + description: Integrates Kubernetes workloads with OpenBao secrets. + - artifact_id: reval-server + plane: control + kind: service-image + requirement: required + description: Revalidates function state in the background. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/helm-reval + - artifact_id: admin-token-issuer-proxy + plane: control + kind: service-image + requirement: required + description: Proxies admin token requests for stack services. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/admin-token-issuer-proxy + - artifact_id: llm-api-gateway + plane: control + kind: service-image + requirement: optional + description: Exposes OpenAI-compatible APIs for LLM functions. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/llm-api-gateway + - artifact_id: stargate + plane: control + kind: service-image + requirement: optional + description: Routes LLM requests to eligible worker instances. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/libraries/rust/stargate + - artifact_id: nvcf-state-metrics-service + plane: control + kind: service-image + requirement: required + description: Exports NVCF resource state as Prometheus metrics. + - artifact_id: nvcf-function-autoscaler + plane: control + kind: service-image + requirement: required + description: Scales functions from NVCF workload metrics. + - artifact_id: nvcf-ui + plane: control + kind: service-image + requirement: optional + description: Serves the optional NVCF administrative interface. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/uis/nvcf-ui + - artifact_id: opentelemetry-collector-contrib + plane: control + kind: service-image + requirement: required + description: Collects and exports NVCF telemetry. + upstream_url: https://github.com/open-telemetry/opentelemetry-collector-contrib + - artifact_id: opentelemetry-operator-image + plane: control + kind: service-image + requirement: required + description: Reconciles OpenTelemetry Collector resources. + upstream_url: https://github.com/open-telemetry/opentelemetry-operator + - artifact_id: victoria-metrics + plane: control + kind: service-image + requirement: required + description: Stores metrics for the default observability profile. + upstream_url: https://github.com/VictoriaMetrics/VictoriaMetrics + - artifact_id: cassandra + plane: control + kind: service-image + requirement: required + description: Stores NVCF account, function, cluster, and service state. + upstream_url: https://github.com/apache/cassandra + - artifact_id: nvcf-cassandra-migrations + plane: control + kind: ea-cve + requirement: required + description: Applies the Cassandra schemas required by Early Access NVCF services. + github_url: https://github.com/NVIDIA/nvcf/tree/main/migrations/cassandra + - artifact_id: helm-nvca-operator + plane: compute + kind: chart + requirement: required + description: Deploys the NVCA operator and compute-plane integration. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nvca-operator + - artifact_id: nvcf-cluster-topology + plane: compute + kind: chart + requirement: required + description: Configures cluster topology resources for compute scheduling. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology + - artifact_id: dynamo-platform + plane: compute + kind: chart + requirement: optional + description: Deploys the optional NVIDIA Dynamo operator. + upstream_url: https://github.com/ai-dynamo/dynamo + - artifact_id: grove-charts + plane: compute + kind: chart + requirement: optional + description: Deploys the optional Grove operator for topology-aware scheduling. + upstream_url: https://github.com/ai-dynamo/grove + - artifact_id: kai-scheduler + plane: compute + kind: chart + requirement: optional + description: Deploys the optional KAI Scheduler. + upstream_url: https://github.com/NVIDIA/KAI-Scheduler + - artifact_id: nvcf-container-cache-chart + plane: compute + kind: chart + requirement: optional + description: Deploys container image caching on GPU cluster nodes. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/container-cache + - name: gpu-operator + version: supported + distribution: https://helm.ngc.nvidia.com/nvidia + plane: compute + kind: chart + requirement: required + description: Manages NVIDIA GPU software on Kubernetes nodes. + upstream_url: https://github.com/NVIDIA/gpu-operator + - name: ebs-csi-driver + version: supported + distribution: https://kubernetes-sigs.github.io/aws-ebs-csi-driver + plane: compute + kind: chart + requirement: optional + description: Provides Amazon EBS persistent volumes for EKS clusters. + upstream_url: https://github.com/kubernetes-sigs/aws-ebs-csi-driver + - name: csi-driver-smb + version: supported + distribution: https://raw.githubusercontent.com/kubernetes-csi/csi-driver-smb/master/charts + plane: compute + kind: chart + requirement: optional + description: Provides SMB persistent volumes for supported deployments. + upstream_url: https://github.com/kubernetes-csi/csi-driver-smb + - name: modelexpress + version: supported + distribution: https://helm.ngc.nvidia.com/nvidia/ai-dynamo + plane: compute + kind: chart + requirement: optional + description: Distributes model weights peer-to-peer between Dynamo workers to reduce scale-out cold starts. Installed separately from the compute-plane stack. + upstream_url: https://github.com/ai-dynamo/modelexpress + - artifact_id: nvca + plane: compute + kind: service-image + requirement: required + description: Registers GPU clusters and orchestrates deployments in-cluster. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/nvca + - artifact_id: nvca-operator + plane: compute + kind: service-image + requirement: required + description: Reconciles NVCA resources and compute-plane configuration. + - artifact_id: nvcf-worker-utils-oss + plane: compute + kind: service-image + requirement: required + description: Proxies NATS traffic between function containers and the control plane. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/worker-utils + - artifact_id: nvcf-worker-init-oss + plane: compute + kind: service-image + requirement: required + description: Prepares function resources before the user container starts. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/worker-init + - artifact_id: nvcf-worker-llm-credentials-oss + plane: compute + kind: service-image + requirement: optional + description: Maintains a current NVCF worker token for LLM function workloads. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/worker-llm-credentials + - artifact_id: crd-upgrader + plane: compute + kind: service-image + requirement: optional + description: Upgrades KAI Scheduler custom resources. + upstream_url: https://github.com/NVIDIA/KAI-Scheduler + - artifact_id: operator + plane: compute + kind: service-image + requirement: optional + description: Reconciles KAI Scheduler resources. + upstream_url: https://github.com/NVIDIA/KAI-Scheduler + - artifact_id: grove-install-crds + plane: compute + kind: service-image + requirement: optional + description: Installs Grove custom resource definitions. + upstream_url: https://github.com/ai-dynamo/grove + - artifact_id: grove-operator + plane: compute + kind: service-image + requirement: optional + description: Reconciles Grove topology-aware scheduling resources. + upstream_url: https://github.com/ai-dynamo/grove + - artifact_id: kubernetes-operator + plane: compute + kind: service-image + requirement: optional + description: Reconciles NVIDIA Dynamo workloads on Kubernetes. + upstream_url: https://github.com/ai-dynamo/dynamo + - artifact_id: nats + plane: compute + kind: service-image + requirement: optional + description: Provides messaging for the optional NVIDIA Dynamo operator. + upstream_url: https://github.com/nats-io/nats-server + - artifact_id: ess-agent + plane: compute + kind: service-image + requirement: required + description: Injects encrypted application secrets into function workloads. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/ess-agent + - artifact_id: nvcf-image-credential-helper + plane: compute + kind: service-image + requirement: required + description: Resolves container image credentials for function workloads. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/image-credential-helper + - name: gpu-operator-validator + version: supported + distribution: https://catalog.ngc.nvidia.com/orgs/nvidia/teams/cloud-native/containers/gpu-operator-validator + plane: compute + kind: service-image + requirement: required + description: Validates GPU Operator components on GPU nodes. + upstream_url: https://github.com/NVIDIA/gpu-operator + - name: k8s-device-plugin + version: supported + distribution: https://catalog.ngc.nvidia.com/orgs/nvidia/teams/k8s/containers/device-plugin + plane: compute + kind: service-image + requirement: required + description: Advertises NVIDIA GPU resources to Kubernetes. + upstream_url: https://github.com/NVIDIA/k8s-device-plugin + - name: modelexpress-server + version: supported + distribution: https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/containers/modelexpress-server + plane: compute + kind: service-image + requirement: optional + description: Serves model weights to Dynamo workers over NIXL RDMA transports. + upstream_url: https://github.com/ai-dynamo/modelexpress + - artifact_id: pylon + plane: compute + kind: service-image + requirement: optional + description: Connects LLM worker pods to the LLM request router. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/libraries/rust/stargate + - artifact_id: nvcf-container-cache-image + plane: compute + kind: service-image + requirement: optional + description: Caches container image layers on GPU cluster nodes. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/container-cache + - artifact_id: nvcf-proxy-tls-certs + plane: compute + kind: service-image + requirement: optional + description: Configures TLS trust for the optional container cache proxy. + - artifact_id: nvcf-self-managed-stack + plane: shared + kind: resource + description: Provides the Helmfile bundle for control-plane deployment. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/self-managed + - artifact_id: nvcf-compute-plane-stack + plane: shared + kind: resource + description: Provides the Helmfile bundle for compute-plane deployment. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/nvcf-compute-plane + - artifact_id: nvcf-observability-stack + plane: shared + kind: resource + description: Provides the Helmfile bundle for standalone observability deployment. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/observability + - artifact_id: nvcf-cli + plane: shared + kind: resource + description: Manages functions, deployments, and clusters from the command line. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/clis/nvcf-cli +release_set: + stacks: + control-plane: + version: 1.0.0 + source_tag: deploy/stacks/self-managed/v1.0.0 + source_commit: 98c440f90a9df76b82be84791a574732b712bce9 + inventory_asset: nvcf-self-managed-stack-inventory.json + documentation_version: "1.0" + status: qualified + compute-plane: + version: 1.0.0 + source_tag: deploy/stacks/nvcf-compute-plane/v1.0.0 + source_commit: 63ac4456d3422508f2ff4b219c7dff8fba1c21c0 + inventory_asset: nvcf-compute-plane-stack-inventory.json + documentation_version: "1.0" + status: qualified + observability: + version: 1.0.0 + source_tag: deploy/stacks/observability/v1.0.0 + source_commit: b77e96a3f8c18e9318b93e697e33e596c4ede7d7 + inventory_asset: nvcf-observability-stack-inventory.json + documentation_version: dev + status: development +compatibility: + - stack: control-plane + train: "1.0" + compatible_with: + compute-plane: 1.0+ + observability: 1.0+ + - stack: compute-plane + train: "1.0" + compatible_with: + control-plane: 1.0+ + observability: 1.0+ + - stack: observability + train: "1.0" + compatible_with: + compute-plane: 1.0+ + control-plane: 1.0+ +stack: + name: nvcf-self-managed-stack + version: 1.0.0 + registry: public-resources + source_tag: deploy/stacks/self-managed/v1.0.0 + source_commit: 98c440f90a9df76b82be84791a574732b712bce9 + pin_sources: + - deploy/stacks/self-managed/global.yaml.gotmpl + - deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl + pin_source_digest: sha256:a4df697e4b90bb69901f3c34e3fa827d121e9a095e2841f17e227b31fed67593 +denylist: + - name: strap + reason: Legacy name replaced by nvcf-service-oss + - name: nv-api-keys + reason: Legacy name replaced by nvcf-api-keys-service + - name: nvcf_worker_niclls + reason: Proprietary image excluded from the published manifest + - name: stargate-client + reason: Renamed worker client deferred to a later release + - name: ddcs-dist-kv + reason: Excluded from the published manifest + - name: ddcs + reason: Excluded from the published manifest + - name: usd-content-cache + reason: Excluded from the published manifest + - name: load_tester_supreme + reason: Excluded from the published manifest + - name: gxcache-webhook + reason: Excluded from the published manifest + - name: gxcache-init + reason: Excluded from the published manifest + - name: gxcache-service + reason: Excluded from the published manifest + - name: helm-gxcache + reason: Excluded from the published manifest + - name: storage-service + reason: Excluded from the published manifest + - name: simple-nginx + reason: Excluded from the published manifest + - name: discovery-service + reason: Excluded from the published manifest + - name: streaming-proxy + reason: Excluded from the published manifest + - name: gdn-streaming + reason: Excluded from the published manifest +artifacts: + - name: k8s + type: image + registry: public-images + upstream_repository: docker.io/alpine/k8s + version: 1.37.0 + stacks: + - self-managed + requirement: required + - name: nats + type: image + registry: public-images + upstream_repository: docker.io/library/nats + version: 2.10.21-alpine + stacks: + - compute-plane + requirement: optional + - name: nats-server-config-reloader + type: image + registry: public-images + upstream_repository: docker.io/natsio/nats-server-config-reloader + version: 0.24.0 + stacks: + - compute-plane + - self-managed + requirement: required + - name: grove-install-crds + type: image + registry: public-images + upstream_repository: ghcr.io/ai-dynamo/grove/grove-install-crds + version: v0.1.0-alpha.12 + stacks: + - compute-plane + requirement: optional + - name: grove-operator + type: image + registry: public-images + upstream_repository: ghcr.io/ai-dynamo/grove/grove-operator + version: v0.1.0-alpha.12 + stacks: + - compute-plane + requirement: optional + - name: crd-upgrader + type: image + registry: public-images + upstream_repository: ghcr.io/kai-scheduler/kai-scheduler/crd-upgrader + version: v0.17.1 + stacks: + - compute-plane + requirement: optional + - name: operator + type: image + registry: public-images + upstream_repository: ghcr.io/kai-scheduler/kai-scheduler/operator + version: v0.17.1 + stacks: + - compute-plane + requirement: optional + - name: opentelemetry-collector-contrib + type: image + registry: public-images + upstream_repository: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib + version: 0.160.0 + stacks: + - observability + requirement: required + - id: opentelemetry-operator-image + name: opentelemetry-operator + type: image + registry: public-images + upstream_repository: ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator + version: 0.158.0 + stacks: + - observability + requirement: required + - name: kubernetes-operator + type: image + registry: public-images + upstream_repository: nvcr.io/nvidia/ai-dynamo/kubernetes-operator + version: 1.4.2 + stacks: + - compute-plane + requirement: optional + - name: admin-token-issuer-proxy + type: image + registry: public-images + version: 1.1.2 + stacks: + - self-managed + requirement: required + - name: cassandra + type: image + registry: public-images + version: 5.0.9-nv-2.0.5 + stacks: + - self-managed + requirement: required + - name: cert-manager-cainjector + type: image + registry: public-images + version: v1.20.2 + stacks: + - self-managed + requirement: required + - name: cert-manager-controller + type: image + registry: public-images + version: v1.20.2 + stacks: + - self-managed + requirement: required + - name: cert-manager-startupapicheck + type: image + registry: public-images + version: v1.20.2 + stacks: + - self-managed + requirement: required + - name: cert-manager-webhook + type: image + registry: public-images + version: v1.20.2 + stacks: + - self-managed + requirement: required + - name: ess-agent + type: image + registry: public-images + version: 1.4.1 + stacks: + - self-managed + requirement: required + - name: icms-service-oss + type: image + registry: public-images + version: 0.7.2 + stacks: + - self-managed + requirement: required + - name: llm-api-gateway + type: image + registry: public-images + version: 0.14.2 + stacks: + - self-managed + requirement: optional + - name: nats-box + type: image + registry: public-images + version: 0.19.7-nonroot + stacks: + - self-managed + requirement: required + - name: nats-server + type: image + registry: public-images + version: 2.14.6-alpine3.22 + stacks: + - self-managed + requirement: required + - name: nvca-operator + type: image + registry: public-images + version: 3.10.0 + stacks: + - compute-plane + requirement: required + - name: nvca + type: image + registry: public-images + version: 3.10.0 + stacks: + - compute-plane + requirement: required + - name: nvcf-ai-api-gateway-service + type: image + registry: public-images + version: 1.35.1 + stacks: + - self-managed + requirement: optional + - name: nvcf-api-keys-service + type: image + registry: public-images + version: 1.9.1 + stacks: + - self-managed + requirement: required + - name: nvcf-cassandra-migrations + type: image + registry: public-images + version: 0.17.6 + stacks: + - self-managed + requirement: required + - name: nvcf-ess + type: image + registry: public-images + version: 0.5.0 + stacks: + - self-managed + requirement: required + - name: nvcf-function-autoscaler + type: image + registry: public-images + version: 1.21.8 + stacks: + - self-managed + requirement: required + - name: nvcf-grpc-proxy + type: image + registry: public-images + version: 1.33.5 + stacks: + - self-managed + requirement: required + - name: nvcf-image-credential-helper + type: image + registry: public-images + version: 0.11.1 + stacks: + - compute-plane + requirement: required + - name: nvcf-invocation-service + type: image + registry: public-images + version: 0.12.1 + stacks: + - self-managed + requirement: required + - name: nvcf-nats-auth-callout-service + type: image + registry: public-images + version: 0.8.3 + stacks: + - self-managed + requirement: required + - name: nvcf-notary + type: image + registry: public-images + version: 1.14.1 + stacks: + - self-managed + requirement: required + - name: nvcf-openbao-migrations + type: image + registry: public-images + version: 0.19.5 + stacks: + - self-managed + requirement: required + - name: nvcf-openbao + type: image + registry: public-images + version: 2.6.2-nv-1.3.4 + stacks: + - self-managed + requirement: required + - name: nvcf-ratelimiter + type: image + registry: public-images + version: 1.17.3 + stacks: + - self-managed + requirement: required + - name: nvcf-service-oss + type: image + registry: public-images + version: 1.18.0 + stacks: + - self-managed + requirement: required + - name: nvcf-state-metrics-service + type: image + registry: public-images + version: 1.23.7 + stacks: + - self-managed + requirement: required + - name: nvcf-ui + type: image + registry: public-images + version: 1.1.2 + stacks: + - self-managed + requirement: optional + - name: nvcf-worker-init-oss + type: image + registry: public-images + version: 1.2.1 + stacks: + - self-managed + requirement: required + - name: nvcf-worker-llm-credentials-oss + type: image + registry: public-images + version: 1.1.2 + stacks: + - self-managed + requirement: required + - name: nvcf-worker-utils-oss + type: image + registry: public-images + version: 1.2.3 + stacks: + - self-managed + requirement: required + - name: nvct-service-oss + type: image + registry: public-images + version: 1.66.0 + stacks: + - self-managed + requirement: required + - name: oss-vault-k8s + type: image + registry: public-images + version: 1.7.4 + stacks: + - self-managed + requirement: required + - name: reval-server + type: image + registry: public-images + version: 0.20.2 + stacks: + - self-managed + requirement: required + - name: stargate + type: image + registry: public-images + version: 0.18.0 + stacks: + - self-managed + requirement: optional + - name: cert-manager-acmesolver + type: image + registry: public-images + upstream_repository: quay.io/jetstack/cert-manager-acmesolver + version: v1.20.2 + stacks: + - self-managed + requirement: optional + - name: victoria-metrics + type: image + registry: public-images + upstream_repository: docker.io/victoriametrics/victoria-metrics + version: v1.150.0 + stacks: + - observability + requirement: required + - name: nvcf-cluster-topology + type: chart + registry: public-helm + version: 0.1.0 + stacks: + - compute-plane + requirement: required + - name: nvcf-default-monitors + type: chart + registry: public-helm + version: 0.2.0 + stacks: + - observability + requirement: required + - name: nvcf-otel-collector + type: chart + registry: public-helm + version: 0.2.0 + stacks: + - observability + requirement: required + - name: dynamo-platform + type: chart + registry: public-helm + upstream_repository: https://helm.ngc.nvidia.com/nvidia/ai-dynamo + version: 1.4.2 + stacks: + - compute-plane + requirement: optional + - name: helm-admin-token-issuer-proxy + type: chart + registry: public-helm + version: 1.5.3 + stacks: + - self-managed + requirement: required + - name: helm-nvca-operator + type: chart + registry: public-helm + version: 1.28.0 + stacks: + - compute-plane + requirement: required + - name: helm-nvcf-api-keys + type: chart + registry: public-helm + version: 1.8.0 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-api + type: chart + registry: public-helm + version: 1.27.1 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-cassandra + type: chart + registry: public-helm + version: 0.21.3 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-cert-manager + type: chart + registry: public-helm + version: 0.1.0 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-ess-api + type: chart + registry: public-helm + version: 1.8.2 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-function-autoscaler + type: chart + registry: public-helm + version: 0.5.2 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-grpc-proxy + type: chart + registry: public-helm + version: 1.7.4 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-invocation-service + type: chart + registry: public-helm + version: 1.6.1 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-llm-api-gateway + type: chart + registry: public-helm + version: 1.4.3 + stacks: + - self-managed + requirement: optional + - name: helm-nvcf-llm-request-router + type: chart + registry: public-helm + version: 1.14.1 + stacks: + - self-managed + requirement: optional + - name: helm-nvcf-nats-auth-callout-service + type: chart + registry: public-helm + version: 1.2.1 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-nats + type: chart + registry: public-helm + version: 0.8.4 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-notary-service + type: chart + registry: public-helm + version: 1.6.0 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-nvct-api + type: chart + registry: public-helm + version: 1.6.0 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-openbao-server + type: chart + registry: public-helm + version: 0.32.6 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-pki + type: chart + registry: public-helm + version: 0.1.0 + stacks: + - self-managed + requirement: optional + - name: helm-nvcf-rate-limiter + type: chart + registry: public-helm + version: 1.2.1 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-sis + type: chart + registry: public-helm + version: 2.4.0 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-state-metrics + type: chart + registry: public-helm + version: 1.0.2 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-ui + type: chart + registry: public-helm + version: 1.1.2 + stacks: + - self-managed + requirement: optional + - name: helm-nvcf-vanity-gateway + type: chart + registry: public-helm + version: 0.5.0 + stacks: + - self-managed + requirement: optional + - name: helm-reval + type: chart + registry: public-helm + version: 1.4.1 + stacks: + - self-managed + requirement: required + - name: nvcf-gateway-routes + type: chart + registry: public-helm + version: 1.18.2 + stacks: + - self-managed + requirement: required + - id: opentelemetry-operator-chart + name: opentelemetry-operator + type: chart + registry: public-helm + upstream_repository: https://open-telemetry.github.io/opentelemetry-helm-charts + version: 0.122.0 + stacks: + - observability + requirement: required + - name: prometheus-operator-crds + type: chart + registry: public-helm + upstream_repository: https://prometheus-community.github.io/helm-charts + version: 31.0.1 + stacks: + - observability + requirement: required + - name: victoria-metrics-single + type: chart + registry: public-helm + upstream_repository: https://victoriametrics.github.io/helm-charts + version: 0.45.0 + stacks: + - observability + requirement: required + - name: grove-charts + type: chart + registry: public-helm + upstream_repository: oci://ghcr.io/ai-dynamo/grove + version: v0.1.0-alpha.12 + stacks: + - compute-plane + requirement: optional + - name: kai-scheduler + type: chart + registry: public-helm + upstream_repository: oci://ghcr.io/kai-scheduler/kai-scheduler + version: v0.17.1 + stacks: + - compute-plane + requirement: optional + - name: pylon + type: image + registry: public-images + version: 0.18.0 +supplemental_artifacts: + - name: nvcf-cli + type: resource + registry: public-resources + version: 1.16.2 + - name: nvcf-compute-plane-stack + type: resource + registry: public-resources + version: 1.0.0 + stacks: + - compute-plane + - name: nvcf-observability-stack + type: resource + registry: public-resources + version: 1.0.0 + stacks: + - observability + - name: nvcf-observability-reference-stack + type: chart + registry: public-helm + version: 1.10.0 + - name: nvcf-example-dashboards + type: chart + registry: public-helm + version: 1.6.0 + - id: nvcf-container-cache-image + name: nvcf-container-cache + type: image + registry: public-images + version: v1.1.36 + - id: nvcf-container-cache-chart + name: nvcf-container-cache + type: chart + registry: public-helm + version: 0.25.22 + - name: nvcf-proxy-tls-certs + type: image + registry: public-images + version: v1.2.10 +outputs: + - path: docs/overview/compatibility-matrix.md + blocks: + - marker: compatibility-matrix + renderer: compatibility-matrix + - path: docs/overview/manifest.md + blocks: + - marker: manifest-artifact-registry-paths + renderer: manifest-artifact-registry-paths + - path: docs/overview/image-mirroring.md + blocks: + - marker: image-mirroring-resource-examples + renderer: image-mirroring-resource-examples + - marker: image-mirroring-stack-snippet + renderer: image-mirroring-stack-snippet + - marker: image-mirroring-compute-stack-snippet + renderer: image-mirroring-compute-stack-snippet + - marker: image-mirroring-observability-stack-snippet + renderer: image-mirroring-observability-stack-snippet + - marker: image-mirroring-cli-snippet + renderer: image-mirroring-cli-snippet + - path: docs/compute-plane/cluster-management/self-managed.md + - path: docs/compute-plane/cluster-management/reference.md diff --git a/docs/version-catalog/main.yaml b/docs/version-catalog/main.yaml index 3ce0f0af27..87897f9c06 100644 --- a/docs/version-catalog/main.yaml +++ b/docs/version-catalog/main.yaml @@ -318,14 +318,6 @@ publications: type: resource version: 1.16.2 registry: public-resources - - name: nvcf-compute-plane-stack - type: resource - version: 0.4.4 - registry: public-resources - - name: nvcf-observability-stack - type: resource - version: 0.2.2 - registry: public-resources version_overrides: - name: nvcf-cli type: resource @@ -333,6 +325,8 @@ version_overrides: source: independent source release; not pinned by stack publication_pending: - helm-nvcf-state-metrics + - nvcf-compute-plane-stack + - nvcf-observability-stack - nvcf-self-managed-stack - nvcf-state-metrics-service manifest: @@ -715,9 +709,9 @@ manifest: upstream_url: https://github.com/apache/cassandra - artifact_id: nvcf-cassandra-migrations plane: control - kind: ea-cve + kind: service-image requirement: required - description: Applies the Cassandra schemas required by Early Access NVCF services. + description: Applies the Cassandra schemas required by NVCF services. github_url: https://github.com/NVIDIA/nvcf/tree/main/migrations/cassandra - artifact_id: helm-nvca-operator plane: compute @@ -926,30 +920,50 @@ manifest: description: Manages functions, deployments, and clusters from the command line. github_url: https://github.com/NVIDIA/nvcf/tree/main/src/clis/nvcf-cli release_set: - documentation_version: dev - status: development stacks: control-plane: - version: 0.20.7 - source_tag: deploy/stacks/self-managed/v0.20.7 - source_commit: c1dd7c6db3eb89c6ff1664b1094a97d7c8e03914 + version: 1.0.0 + source_tag: deploy/stacks/self-managed/v1.0.0 + source_commit: 98c440f90a9df76b82be84791a574732b712bce9 inventory_asset: nvcf-self-managed-stack-inventory.json + documentation_version: "1.0" + status: qualified compute-plane: - version: 0.4.4 - source_tag: deploy/stacks/nvcf-compute-plane/v0.4.4 + version: 1.0.0 + source_tag: deploy/stacks/nvcf-compute-plane/v1.0.0 source_commit: 63ac4456d3422508f2ff4b219c7dff8fba1c21c0 inventory_asset: nvcf-compute-plane-stack-inventory.json + documentation_version: "1.0" + status: qualified observability: - version: 0.2.2 - source_tag: deploy/stacks/observability/v0.2.2 + version: 1.0.0 + source_tag: deploy/stacks/observability/v1.0.0 source_commit: b77e96a3f8c18e9318b93e697e33e596c4ede7d7 inventory_asset: nvcf-observability-stack-inventory.json + documentation_version: "1.0" + status: qualified +compatibility: + - stack: control-plane + train: "1.0" + compatible_with: + compute-plane: 1.0+ + observability: 1.0+ + - stack: compute-plane + train: "1.0" + compatible_with: + control-plane: 1.0+ + observability: 1.0+ + - stack: observability + train: "1.0" + compatible_with: + compute-plane: 1.0+ + control-plane: 1.0+ stack: name: nvcf-self-managed-stack - version: 0.20.7 + version: 1.0.0 registry: public-resources - source_tag: deploy/stacks/self-managed/v0.20.7 - source_commit: c1dd7c6db3eb89c6ff1664b1094a97d7c8e03914 + source_tag: deploy/stacks/self-managed/v1.0.0 + source_commit: 98c440f90a9df76b82be84791a574732b712bce9 pin_sources: - deploy/stacks/self-managed/global.yaml.gotmpl - deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl @@ -1264,7 +1278,7 @@ artifacts: - name: nvcf-state-metrics-service type: image registry: public-images - version: 1.24.1 + version: 1.23.7 stacks: - self-managed requirement: required @@ -1512,7 +1526,7 @@ artifacts: - name: helm-nvcf-state-metrics type: chart registry: public-helm - version: 1.0.6 + version: 1.0.2 stacks: - self-managed requirement: required @@ -1597,13 +1611,13 @@ supplemental_artifacts: - name: nvcf-compute-plane-stack type: resource registry: public-resources - version: 0.4.4 + version: 1.0.0 stacks: - compute-plane - name: nvcf-observability-stack type: resource registry: public-resources - version: 0.2.2 + version: 1.0.0 stacks: - observability - name: nvcf-observability-reference-stack @@ -1629,11 +1643,15 @@ supplemental_artifacts: registry: public-images version: v1.2.10 outputs: - - path: docs/user/manifest.md + - path: docs/overview/compatibility-matrix.md + blocks: + - marker: compatibility-matrix + renderer: compatibility-matrix + - path: docs/overview/manifest.md blocks: - marker: manifest-artifact-registry-paths renderer: manifest-artifact-registry-paths - - path: docs/user/image-mirroring.md + - path: docs/overview/image-mirroring.md blocks: - marker: image-mirroring-resource-examples renderer: image-mirroring-resource-examples @@ -1645,5 +1663,5 @@ outputs: renderer: image-mirroring-observability-stack-snippet - marker: image-mirroring-cli-snippet renderer: image-mirroring-cli-snippet - - path: docs/user/cluster-management/self-managed.md - - path: docs/user/cluster-management/reference.md + - path: docs/compute-plane/cluster-management/self-managed.md + - path: docs/compute-plane/cluster-management/reference.md diff --git a/docs/version-catalog/observability-1.0.yaml b/docs/version-catalog/observability-1.0.yaml new file mode 100644 index 0000000000..038971c94a --- /dev/null +++ b/docs/version-catalog/observability-1.0.yaml @@ -0,0 +1,1667 @@ +version: 1 +target: main +registries: + public-helm: + host: https://helm.ngc.nvidia.com + namespace: nvidia/nvcf + repository_alias: nvcf + public-images: + host: nvcr.io + namespace: nvidia/nvcf + public-resources: + host: nvcr.io + namespace: nvidia/nvcf +publications: + - name: cert-manager-cainjector + type: image + version: v1.20.2 + registry: public-images + - name: cert-manager-controller + type: image + version: v1.20.2 + registry: public-images + - name: cert-manager-startupapicheck + type: image + version: v1.20.2 + registry: public-images + - name: cert-manager-webhook + type: image + version: v1.20.2 + registry: public-images + - name: oss-vault-k8s + type: image + version: 1.7.4 + registry: public-images + - name: nats-box + type: image + version: 0.19.7-nonroot + registry: public-images + - name: helm-nvcf-cert-manager + type: chart + version: 0.1.0 + registry: public-helm + chart_format: http + - name: helm-nvcf-pki + type: chart + version: 0.1.0 + registry: public-helm + chart_format: http + - name: helm-nvcf-ui + type: chart + version: 1.1.2 + registry: public-helm + chart_format: http + - name: nvcf-container-cache + type: image + version: v1.1.36 + registry: public-images + - name: nvcf-proxy-tls-certs + type: image + version: v1.2.10 + registry: public-images + - name: nvcf-container-cache + type: chart + version: 0.25.22 + registry: public-helm + chart_format: http + - name: nvcf-example-dashboards + type: chart + version: 1.6.0 + registry: public-helm + chart_format: http + - name: nvcf-observability-reference-stack + type: chart + version: 1.10.0 + registry: public-helm + chart_format: http + - name: admin-token-issuer-proxy + type: image + version: 1.1.2 + registry: public-images + - name: cassandra + type: image + version: 5.0.9-nv-2.0.5 + registry: public-images + - name: ess-agent + type: image + version: 1.4.1 + registry: public-images + - name: nats-server + type: image + version: 2.14.6-alpine3.22 + registry: public-images + - name: nvcf-api-keys-service + type: image + version: 1.9.1 + registry: public-images + - name: nvcf-cassandra-migrations + type: image + version: 0.17.6 + registry: public-images + - name: nvcf-image-credential-helper + type: image + version: 0.11.1 + registry: public-images + - name: nvcf-invocation-service + type: image + version: 0.12.1 + registry: public-images + - name: nvcf-nats-auth-callout-service + type: image + version: 0.8.3 + registry: public-images + - name: nvcf-openbao + type: image + version: 2.6.2-nv-1.3.4 + registry: public-images + - name: nvcf-openbao-migrations + type: image + version: 0.19.5 + registry: public-images + - name: nvcf-service-oss + type: image + version: 1.18.0 + registry: public-images + - name: nvcf-ui + type: image + version: 1.1.2 + registry: public-images + - name: nvcf-worker-init-oss + type: image + version: 1.2.1 + registry: public-images + - name: nvcf-worker-llm-credentials-oss + type: image + version: 1.1.2 + registry: public-images + - name: nvcf-worker-utils-oss + type: image + version: 1.2.3 + registry: public-images + - name: nvct-service-oss + type: image + version: 1.66.0 + registry: public-images + - name: reval-server + type: image + version: 0.20.2 + registry: public-images + - name: stargate + type: image + version: 0.18.0 + registry: public-images + - name: pylon + type: image + version: 0.18.0 + registry: public-images + - name: icms-service-oss + type: image + version: 0.7.2 + registry: public-images + - name: nvca-operator + type: image + version: 3.10.0 + registry: public-images + - name: nvca + type: image + version: 3.10.0 + registry: public-images + - name: llm-api-gateway + type: image + version: 0.14.2 + registry: public-images + - name: nvcf-ai-api-gateway-service + type: image + version: 1.35.1 + registry: public-images + - name: nvcf-ess + type: image + version: 0.5.0 + registry: public-images + - name: nvcf-function-autoscaler + type: image + version: 1.21.8 + registry: public-images + - name: nvcf-grpc-proxy + type: image + version: 1.33.5 + registry: public-images + - name: nvcf-notary + type: image + version: 1.14.1 + registry: public-images + - name: nvcf-ratelimiter + type: image + version: 1.17.3 + registry: public-images + - name: helm-admin-token-issuer-proxy + type: chart + version: 1.5.3 + registry: public-helm + chart_format: http + - name: helm-nvca-operator + type: chart + version: 1.28.0 + registry: public-helm + chart_format: http + - name: helm-nvcf-api + type: chart + version: 1.27.1 + registry: public-helm + chart_format: http + - name: helm-nvcf-api-keys + type: chart + version: 1.8.0 + registry: public-helm + chart_format: http + - name: helm-nvcf-cassandra + type: chart + version: 0.21.3 + registry: public-helm + chart_format: http + - name: helm-nvcf-ess-api + type: chart + version: 1.8.2 + registry: public-helm + chart_format: http + - name: helm-nvcf-function-autoscaler + type: chart + version: 0.5.2 + registry: public-helm + chart_format: http + - name: helm-nvcf-invocation-service + type: chart + version: 1.6.1 + registry: public-helm + chart_format: http + - name: helm-nvcf-llm-api-gateway + type: chart + version: 1.4.3 + registry: public-helm + chart_format: http + - name: helm-nvcf-nats + type: chart + version: 0.8.4 + registry: public-helm + chart_format: http + - name: helm-nvcf-nats-auth-callout-service + type: chart + version: 1.2.1 + registry: public-helm + chart_format: http + - name: helm-nvcf-notary-service + type: chart + version: 1.6.0 + registry: public-helm + chart_format: http + - name: helm-nvcf-nvct-api + type: chart + version: 1.6.0 + registry: public-helm + chart_format: http + - name: helm-nvcf-openbao-server + type: chart + version: 0.32.6 + registry: public-helm + chart_format: http + - name: helm-nvcf-rate-limiter + type: chart + version: 1.2.1 + registry: public-helm + chart_format: http + - name: helm-nvcf-sis + type: chart + version: 2.4.0 + registry: public-helm + chart_format: http + - name: helm-nvcf-vanity-gateway + type: chart + version: 0.5.0 + registry: public-helm + chart_format: http + - name: helm-reval + type: chart + version: 1.4.1 + registry: public-helm + chart_format: http + - name: helm-nvcf-grpc-proxy + type: chart + version: 1.7.4 + registry: public-helm + chart_format: http + - name: helm-nvcf-llm-request-router + type: chart + version: 1.14.1 + registry: public-helm + chart_format: http + - name: nvcf-cluster-topology + type: chart + version: 0.1.0 + registry: public-helm + chart_format: http + - name: nvcf-default-monitors + type: chart + version: 0.2.0 + registry: public-helm + chart_format: http + - name: nvcf-gateway-routes + type: chart + version: 1.18.2 + registry: public-helm + chart_format: http + - name: nvcf-otel-collector + type: chart + version: 0.2.0 + registry: public-helm + chart_format: http + - name: nvcf-cli + type: resource + version: 1.16.2 + registry: public-resources +version_overrides: + - name: nvcf-cli + type: resource + version: 1.16.2 + source: independent source release; not pinned by stack +publication_pending: + - helm-nvcf-state-metrics + - nvcf-compute-plane-stack + - nvcf-observability-stack + - nvcf-self-managed-stack + - nvcf-state-metrics-service +manifest: + entries: + - artifact_id: helm-nvcf-api-keys + plane: control + kind: chart + requirement: required + description: Deploys the API key management service. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/api-keys-colocated + - artifact_id: helm-nvcf-api + plane: control + kind: chart + requirement: required + description: Deploys the NVCF API service. + - artifact_id: helm-nvcf-cassandra + plane: control + kind: chart + requirement: required + description: Deploys Cassandra and its initialization jobs. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/cassandra + upstream_url: https://github.com/bitnami/charts/tree/main/bitnami/cassandra + - artifact_id: helm-nvcf-cert-manager + plane: control + kind: chart + requirement: required + description: Deploys the NVCF cert-manager configuration. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/cert-manager + upstream_url: https://github.com/cert-manager/cert-manager + - artifact_id: helm-nvcf-ess-api + plane: control + kind: chart + requirement: required + description: Deploys the Encrypted Secrets Service API. + - artifact_id: helm-nvcf-function-autoscaler + plane: control + kind: chart + requirement: required + description: Deploys the function autoscaler for observability-driven scaling. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/function-autoscaler + - artifact_id: helm-nvcf-grpc-proxy + plane: control + kind: chart + requirement: required + description: Deploys the gRPC proxy service. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/grpc-proxy + - artifact_id: helm-nvcf-invocation-service + plane: control + kind: chart + requirement: required + description: Deploys the HTTP invocation service. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/http-invocation + - artifact_id: helm-nvcf-nats-auth-callout-service + plane: control + kind: chart + requirement: required + description: Deploys the NATS authorization callout service. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nats-auth-callout + - artifact_id: helm-nvcf-nats + plane: control + kind: chart + requirement: required + description: Deploys NATS messaging for the control plane. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nats + upstream_url: https://github.com/nats-io/k8s + - artifact_id: helm-nvcf-notary-service + plane: control + kind: chart + requirement: required + description: Deploys the notary service for signing and validation. + - artifact_id: helm-nvcf-nvct-api + plane: control + kind: chart + requirement: required + description: Deploys the NVCF tenant API service. + - artifact_id: helm-nvcf-openbao-server + plane: control + kind: chart + requirement: required + description: Deploys OpenBao secret management. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/openbao + upstream_url: https://github.com/openbao/openbao-helm + - artifact_id: helm-nvcf-sis + plane: control + kind: chart + requirement: required + description: Deploys the Spot Instance Service. + - artifact_id: helm-reval + plane: control + kind: chart + requirement: required + description: Deploys the function revalidation service. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/helm-reval + - artifact_id: helm-admin-token-issuer-proxy + plane: control + kind: chart + requirement: required + description: Deploys the admin token issuer proxy. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/admin-token-issuer-proxy + - artifact_id: helm-nvcf-llm-api-gateway + plane: control + kind: chart + requirement: optional + description: Deploys the OpenAI-compatible LLM API gateway. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/llm-api-gateway + - artifact_id: helm-nvcf-llm-request-router + plane: control + kind: chart + requirement: optional + description: Deploys the LLM request router. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/llm-request-router + - artifact_id: helm-nvcf-pki + plane: control + kind: chart + requirement: optional + description: Provisions the OpenBao-backed ClusterIssuer for NVCF service TLS. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nvcf-pki + - artifact_id: helm-nvcf-rate-limiter + plane: control + kind: chart + requirement: required + description: Deploys request rate limiting for supported invocation paths. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/ratelimiter + - artifact_id: helm-nvcf-state-metrics + plane: control + kind: chart + requirement: required + description: Deploys NVCF state metrics for observability. + - artifact_id: helm-nvcf-ui + plane: control + kind: chart + requirement: optional + description: Deploys the optional NVCF UI admin panel. + - artifact_id: helm-nvcf-vanity-gateway + plane: control + kind: chart + requirement: optional + description: Deploys the optional vanity hostname gateway. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/vanity-gateway + - artifact_id: nvcf-example-dashboards + plane: control + kind: chart + requirement: optional + description: Deploys example Grafana dashboards for NVCF telemetry. + - artifact_id: nvcf-gateway-routes + plane: control + kind: chart + requirement: required + description: Deploys Gateway API routes for NVCF services. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/gateway-routes + - artifact_id: nvcf-default-monitors + plane: control + kind: chart + requirement: required + description: Deploys the default service and pod monitors for NVCF. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/observability/charts/nvcf-default-monitors + - artifact_id: nvcf-otel-collector + plane: control + kind: chart + requirement: required + description: Configures the OpenTelemetry Collector used by NVCF. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/observability/charts/nvcf-otel-collector + - artifact_id: opentelemetry-operator-chart + plane: control + kind: chart + requirement: required + description: Deploys the OpenTelemetry Operator. + upstream_url: https://github.com/open-telemetry/opentelemetry-operator + - artifact_id: prometheus-operator-crds + plane: control + kind: chart + requirement: required + description: Installs the Prometheus Operator custom resource definitions. + upstream_url: https://github.com/prometheus-community/helm-charts + - artifact_id: victoria-metrics-single + plane: control + kind: chart + requirement: required + description: Deploys the default metrics storage backend. + upstream_url: https://github.com/VictoriaMetrics/helm-charts + - artifact_id: nvcf-observability-reference-stack + plane: control + kind: chart + requirement: optional + description: Deploys a reference observability backend for evaluation. + - artifact_id: k8s + plane: control + kind: service-image + requirement: required + description: Provides Kubernetes command-line utilities for deployment jobs. + github_url: https://github.com/alpine-docker/k8s + - artifact_id: cert-manager-acmesolver + plane: control + kind: service-image + requirement: optional + description: Serves temporary ACME HTTP-01 domain-validation challenges. + upstream_url: https://github.com/cert-manager/cert-manager + - artifact_id: cert-manager-cainjector + plane: control + kind: service-image + requirement: required + description: Injects certificate authority data into Kubernetes resources. + upstream_url: https://github.com/cert-manager/cert-manager + - artifact_id: cert-manager-controller + plane: control + kind: service-image + requirement: required + description: Reconciles certificates and issuers for the control plane. + upstream_url: https://github.com/cert-manager/cert-manager + - artifact_id: cert-manager-startupapicheck + plane: control + kind: service-image + requirement: required + description: Verifies that the cert-manager API is ready. + upstream_url: https://github.com/cert-manager/cert-manager + - artifact_id: cert-manager-webhook + plane: control + kind: service-image + requirement: required + description: Validates and converts cert-manager API resources. + upstream_url: https://github.com/cert-manager/cert-manager + - artifact_id: nvcf-ess + plane: control + kind: service-image + requirement: required + description: Provides encrypted application secrets to NVCF workloads. + - artifact_id: nats-box + plane: control + kind: service-image + requirement: required + description: Provides NATS administration and diagnostic utilities. + upstream_url: https://github.com/nats-io/nats-box + - artifact_id: nats-server + plane: control + kind: service-image + requirement: required + description: Provides messaging for function deployment and invocation. + upstream_url: https://github.com/nats-io/nats-server + - artifact_id: nats-server-config-reloader + plane: shared + kind: service-image + requirement: required + description: Reloads NATS configuration for the control plane and optional NVIDIA Dynamo deployment. + upstream_url: https://github.com/nats-io/k8s + - artifact_id: nvcf-notary + plane: control + kind: service-image + requirement: required + description: Signs and validates functions and cluster nodes. + - artifact_id: nvcf-ai-api-gateway-service + plane: control + kind: service-image + requirement: optional + description: Serves the optional vanity hostname gateway. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/vanity-gateway + - artifact_id: nvcf-api-keys-service + plane: control + kind: service-image + requirement: required + description: Creates and manages NVCF API keys. + - artifact_id: icms-service-oss + plane: control + kind: service-image + requirement: required + description: Manages instance and cluster lifecycle operations. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/instance-cluster-management + - artifact_id: nvcf-grpc-proxy + plane: control + kind: service-image + requirement: required + description: Proxies bidirectional gRPC traffic between the control and compute planes. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/grpc-proxy + - artifact_id: nvcf-invocation-service + plane: control + kind: service-image + requirement: required + description: Routes stateless HTTP function invocation requests. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/http-invocation + - artifact_id: nvcf-nats-auth-callout-service + plane: control + kind: service-image + requirement: required + description: Authorizes NATS clients for NVCF services and workloads. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/nats-auth-callout + - artifact_id: nvcf-openbao + plane: control + kind: service-image + requirement: required + description: Stores and manages control-plane secrets. + upstream_url: https://github.com/openbao/openbao + - artifact_id: nvcf-openbao-migrations + plane: control + kind: service-image + requirement: required + description: Applies the OpenBao configuration required by NVCF. + github_url: https://github.com/NVIDIA/nvcf/tree/main/migrations/openbao + - artifact_id: nvcf-ratelimiter + plane: control + kind: service-image + requirement: required + description: Enforces request rate limits for supported invocation paths. + - artifact_id: nvcf-service-oss + plane: control + kind: service-image + requirement: required + description: Provides the primary NVCF control-plane API. + - artifact_id: nvct-service-oss + plane: control + kind: service-image + requirement: required + description: Provides tenant-scoped NVCF control-plane operations. + - artifact_id: oss-vault-k8s + plane: control + kind: service-image + requirement: required + description: Integrates Kubernetes workloads with OpenBao secrets. + - artifact_id: reval-server + plane: control + kind: service-image + requirement: required + description: Revalidates function state in the background. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/helm-reval + - artifact_id: admin-token-issuer-proxy + plane: control + kind: service-image + requirement: required + description: Proxies admin token requests for stack services. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/control-plane-services/admin-token-issuer-proxy + - artifact_id: llm-api-gateway + plane: control + kind: service-image + requirement: optional + description: Exposes OpenAI-compatible APIs for LLM functions. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/invocation-plane-services/llm-api-gateway + - artifact_id: stargate + plane: control + kind: service-image + requirement: optional + description: Routes LLM requests to eligible worker instances. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/libraries/rust/stargate + - artifact_id: nvcf-state-metrics-service + plane: control + kind: service-image + requirement: required + description: Exports NVCF resource state as Prometheus metrics. + - artifact_id: nvcf-function-autoscaler + plane: control + kind: service-image + requirement: required + description: Scales functions from NVCF workload metrics. + - artifact_id: nvcf-ui + plane: control + kind: service-image + requirement: optional + description: Serves the optional NVCF administrative interface. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/uis/nvcf-ui + - artifact_id: opentelemetry-collector-contrib + plane: control + kind: service-image + requirement: required + description: Collects and exports NVCF telemetry. + upstream_url: https://github.com/open-telemetry/opentelemetry-collector-contrib + - artifact_id: opentelemetry-operator-image + plane: control + kind: service-image + requirement: required + description: Reconciles OpenTelemetry Collector resources. + upstream_url: https://github.com/open-telemetry/opentelemetry-operator + - artifact_id: victoria-metrics + plane: control + kind: service-image + requirement: required + description: Stores metrics for the default observability profile. + upstream_url: https://github.com/VictoriaMetrics/VictoriaMetrics + - artifact_id: cassandra + plane: control + kind: service-image + requirement: required + description: Stores NVCF account, function, cluster, and service state. + upstream_url: https://github.com/apache/cassandra + - artifact_id: nvcf-cassandra-migrations + plane: control + kind: ea-cve + requirement: required + description: Applies the Cassandra schemas required by Early Access NVCF services. + github_url: https://github.com/NVIDIA/nvcf/tree/main/migrations/cassandra + - artifact_id: helm-nvca-operator + plane: compute + kind: chart + requirement: required + description: Deploys the NVCA operator and compute-plane integration. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nvca-operator + - artifact_id: nvcf-cluster-topology + plane: compute + kind: chart + requirement: required + description: Configures cluster topology resources for compute scheduling. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology + - artifact_id: dynamo-platform + plane: compute + kind: chart + requirement: optional + description: Deploys the optional NVIDIA Dynamo operator. + upstream_url: https://github.com/ai-dynamo/dynamo + - artifact_id: grove-charts + plane: compute + kind: chart + requirement: optional + description: Deploys the optional Grove operator for topology-aware scheduling. + upstream_url: https://github.com/ai-dynamo/grove + - artifact_id: kai-scheduler + plane: compute + kind: chart + requirement: optional + description: Deploys the optional KAI Scheduler. + upstream_url: https://github.com/NVIDIA/KAI-Scheduler + - artifact_id: nvcf-container-cache-chart + plane: compute + kind: chart + requirement: optional + description: Deploys container image caching on GPU cluster nodes. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/container-cache + - name: gpu-operator + version: supported + distribution: https://helm.ngc.nvidia.com/nvidia + plane: compute + kind: chart + requirement: required + description: Manages NVIDIA GPU software on Kubernetes nodes. + upstream_url: https://github.com/NVIDIA/gpu-operator + - name: ebs-csi-driver + version: supported + distribution: https://kubernetes-sigs.github.io/aws-ebs-csi-driver + plane: compute + kind: chart + requirement: optional + description: Provides Amazon EBS persistent volumes for EKS clusters. + upstream_url: https://github.com/kubernetes-sigs/aws-ebs-csi-driver + - name: csi-driver-smb + version: supported + distribution: https://raw.githubusercontent.com/kubernetes-csi/csi-driver-smb/master/charts + plane: compute + kind: chart + requirement: optional + description: Provides SMB persistent volumes for supported deployments. + upstream_url: https://github.com/kubernetes-csi/csi-driver-smb + - name: modelexpress + version: supported + distribution: https://helm.ngc.nvidia.com/nvidia/ai-dynamo + plane: compute + kind: chart + requirement: optional + description: Distributes model weights peer-to-peer between Dynamo workers to reduce scale-out cold starts. Installed separately from the compute-plane stack. + upstream_url: https://github.com/ai-dynamo/modelexpress + - artifact_id: nvca + plane: compute + kind: service-image + requirement: required + description: Registers GPU clusters and orchestrates deployments in-cluster. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/nvca + - artifact_id: nvca-operator + plane: compute + kind: service-image + requirement: required + description: Reconciles NVCA resources and compute-plane configuration. + - artifact_id: nvcf-worker-utils-oss + plane: compute + kind: service-image + requirement: required + description: Proxies NATS traffic between function containers and the control plane. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/worker-utils + - artifact_id: nvcf-worker-init-oss + plane: compute + kind: service-image + requirement: required + description: Prepares function resources before the user container starts. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/worker-init + - artifact_id: nvcf-worker-llm-credentials-oss + plane: compute + kind: service-image + requirement: optional + description: Maintains a current NVCF worker token for LLM function workloads. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/worker-llm-credentials + - artifact_id: crd-upgrader + plane: compute + kind: service-image + requirement: optional + description: Upgrades KAI Scheduler custom resources. + upstream_url: https://github.com/NVIDIA/KAI-Scheduler + - artifact_id: operator + plane: compute + kind: service-image + requirement: optional + description: Reconciles KAI Scheduler resources. + upstream_url: https://github.com/NVIDIA/KAI-Scheduler + - artifact_id: grove-install-crds + plane: compute + kind: service-image + requirement: optional + description: Installs Grove custom resource definitions. + upstream_url: https://github.com/ai-dynamo/grove + - artifact_id: grove-operator + plane: compute + kind: service-image + requirement: optional + description: Reconciles Grove topology-aware scheduling resources. + upstream_url: https://github.com/ai-dynamo/grove + - artifact_id: kubernetes-operator + plane: compute + kind: service-image + requirement: optional + description: Reconciles NVIDIA Dynamo workloads on Kubernetes. + upstream_url: https://github.com/ai-dynamo/dynamo + - artifact_id: nats + plane: compute + kind: service-image + requirement: optional + description: Provides messaging for the optional NVIDIA Dynamo operator. + upstream_url: https://github.com/nats-io/nats-server + - artifact_id: ess-agent + plane: compute + kind: service-image + requirement: required + description: Injects encrypted application secrets into function workloads. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/ess-agent + - artifact_id: nvcf-image-credential-helper + plane: compute + kind: service-image + requirement: required + description: Resolves container image credentials for function workloads. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/compute-plane-services/image-credential-helper + - name: gpu-operator-validator + version: supported + distribution: https://catalog.ngc.nvidia.com/orgs/nvidia/teams/cloud-native/containers/gpu-operator-validator + plane: compute + kind: service-image + requirement: required + description: Validates GPU Operator components on GPU nodes. + upstream_url: https://github.com/NVIDIA/gpu-operator + - name: k8s-device-plugin + version: supported + distribution: https://catalog.ngc.nvidia.com/orgs/nvidia/teams/k8s/containers/device-plugin + plane: compute + kind: service-image + requirement: required + description: Advertises NVIDIA GPU resources to Kubernetes. + upstream_url: https://github.com/NVIDIA/k8s-device-plugin + - name: modelexpress-server + version: supported + distribution: https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/containers/modelexpress-server + plane: compute + kind: service-image + requirement: optional + description: Serves model weights to Dynamo workers over NIXL RDMA transports. + upstream_url: https://github.com/ai-dynamo/modelexpress + - artifact_id: pylon + plane: compute + kind: service-image + requirement: optional + description: Connects LLM worker pods to the LLM request router. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/libraries/rust/stargate + - artifact_id: nvcf-container-cache-image + plane: compute + kind: service-image + requirement: optional + description: Caches container image layers on GPU cluster nodes. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/container-cache + - artifact_id: nvcf-proxy-tls-certs + plane: compute + kind: service-image + requirement: optional + description: Configures TLS trust for the optional container cache proxy. + - artifact_id: nvcf-self-managed-stack + plane: shared + kind: resource + description: Provides the Helmfile bundle for control-plane deployment. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/self-managed + - artifact_id: nvcf-compute-plane-stack + plane: shared + kind: resource + description: Provides the Helmfile bundle for compute-plane deployment. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/nvcf-compute-plane + - artifact_id: nvcf-observability-stack + plane: shared + kind: resource + description: Provides the Helmfile bundle for standalone observability deployment. + github_url: https://github.com/NVIDIA/nvcf/tree/main/deploy/stacks/observability + - artifact_id: nvcf-cli + plane: shared + kind: resource + description: Manages functions, deployments, and clusters from the command line. + github_url: https://github.com/NVIDIA/nvcf/tree/main/src/clis/nvcf-cli +release_set: + stacks: + control-plane: + version: 1.0.0 + source_tag: deploy/stacks/self-managed/v1.0.0 + source_commit: 98c440f90a9df76b82be84791a574732b712bce9 + inventory_asset: nvcf-self-managed-stack-inventory.json + documentation_version: "1.0" + status: qualified + compute-plane: + version: 1.0.0 + source_tag: deploy/stacks/nvcf-compute-plane/v1.0.0 + source_commit: 63ac4456d3422508f2ff4b219c7dff8fba1c21c0 + inventory_asset: nvcf-compute-plane-stack-inventory.json + documentation_version: dev + status: development + observability: + version: 1.0.0 + source_tag: deploy/stacks/observability/v1.0.0 + source_commit: b77e96a3f8c18e9318b93e697e33e596c4ede7d7 + inventory_asset: nvcf-observability-stack-inventory.json + documentation_version: "1.0" + status: qualified +compatibility: + - stack: control-plane + train: "1.0" + compatible_with: + compute-plane: 1.0+ + observability: 1.0+ + - stack: compute-plane + train: "1.0" + compatible_with: + control-plane: 1.0+ + observability: 1.0+ + - stack: observability + train: "1.0" + compatible_with: + compute-plane: 1.0+ + control-plane: 1.0+ +stack: + name: nvcf-self-managed-stack + version: 1.0.0 + registry: public-resources + source_tag: deploy/stacks/self-managed/v1.0.0 + source_commit: 98c440f90a9df76b82be84791a574732b712bce9 + pin_sources: + - deploy/stacks/self-managed/global.yaml.gotmpl + - deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl + pin_source_digest: sha256:a4df697e4b90bb69901f3c34e3fa827d121e9a095e2841f17e227b31fed67593 +denylist: + - name: strap + reason: Legacy name replaced by nvcf-service-oss + - name: nv-api-keys + reason: Legacy name replaced by nvcf-api-keys-service + - name: nvcf_worker_niclls + reason: Proprietary image excluded from the published manifest + - name: stargate-client + reason: Renamed worker client deferred to a later release + - name: ddcs-dist-kv + reason: Excluded from the published manifest + - name: ddcs + reason: Excluded from the published manifest + - name: usd-content-cache + reason: Excluded from the published manifest + - name: load_tester_supreme + reason: Excluded from the published manifest + - name: gxcache-webhook + reason: Excluded from the published manifest + - name: gxcache-init + reason: Excluded from the published manifest + - name: gxcache-service + reason: Excluded from the published manifest + - name: helm-gxcache + reason: Excluded from the published manifest + - name: storage-service + reason: Excluded from the published manifest + - name: simple-nginx + reason: Excluded from the published manifest + - name: discovery-service + reason: Excluded from the published manifest + - name: streaming-proxy + reason: Excluded from the published manifest + - name: gdn-streaming + reason: Excluded from the published manifest +artifacts: + - name: k8s + type: image + registry: public-images + upstream_repository: docker.io/alpine/k8s + version: 1.37.0 + stacks: + - self-managed + requirement: required + - name: nats + type: image + registry: public-images + upstream_repository: docker.io/library/nats + version: 2.10.21-alpine + stacks: + - compute-plane + requirement: optional + - name: nats-server-config-reloader + type: image + registry: public-images + upstream_repository: docker.io/natsio/nats-server-config-reloader + version: 0.24.0 + stacks: + - compute-plane + - self-managed + requirement: required + - name: grove-install-crds + type: image + registry: public-images + upstream_repository: ghcr.io/ai-dynamo/grove/grove-install-crds + version: v0.1.0-alpha.12 + stacks: + - compute-plane + requirement: optional + - name: grove-operator + type: image + registry: public-images + upstream_repository: ghcr.io/ai-dynamo/grove/grove-operator + version: v0.1.0-alpha.12 + stacks: + - compute-plane + requirement: optional + - name: crd-upgrader + type: image + registry: public-images + upstream_repository: ghcr.io/kai-scheduler/kai-scheduler/crd-upgrader + version: v0.17.1 + stacks: + - compute-plane + requirement: optional + - name: operator + type: image + registry: public-images + upstream_repository: ghcr.io/kai-scheduler/kai-scheduler/operator + version: v0.17.1 + stacks: + - compute-plane + requirement: optional + - name: opentelemetry-collector-contrib + type: image + registry: public-images + upstream_repository: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib + version: 0.160.0 + stacks: + - observability + requirement: required + - id: opentelemetry-operator-image + name: opentelemetry-operator + type: image + registry: public-images + upstream_repository: ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator + version: 0.158.0 + stacks: + - observability + requirement: required + - name: kubernetes-operator + type: image + registry: public-images + upstream_repository: nvcr.io/nvidia/ai-dynamo/kubernetes-operator + version: 1.4.2 + stacks: + - compute-plane + requirement: optional + - name: admin-token-issuer-proxy + type: image + registry: public-images + version: 1.1.2 + stacks: + - self-managed + requirement: required + - name: cassandra + type: image + registry: public-images + version: 5.0.9-nv-2.0.5 + stacks: + - self-managed + requirement: required + - name: cert-manager-cainjector + type: image + registry: public-images + version: v1.20.2 + stacks: + - self-managed + requirement: required + - name: cert-manager-controller + type: image + registry: public-images + version: v1.20.2 + stacks: + - self-managed + requirement: required + - name: cert-manager-startupapicheck + type: image + registry: public-images + version: v1.20.2 + stacks: + - self-managed + requirement: required + - name: cert-manager-webhook + type: image + registry: public-images + version: v1.20.2 + stacks: + - self-managed + requirement: required + - name: ess-agent + type: image + registry: public-images + version: 1.4.1 + stacks: + - self-managed + requirement: required + - name: icms-service-oss + type: image + registry: public-images + version: 0.7.2 + stacks: + - self-managed + requirement: required + - name: llm-api-gateway + type: image + registry: public-images + version: 0.14.2 + stacks: + - self-managed + requirement: optional + - name: nats-box + type: image + registry: public-images + version: 0.19.7-nonroot + stacks: + - self-managed + requirement: required + - name: nats-server + type: image + registry: public-images + version: 2.14.6-alpine3.22 + stacks: + - self-managed + requirement: required + - name: nvca-operator + type: image + registry: public-images + version: 3.10.0 + stacks: + - compute-plane + requirement: required + - name: nvca + type: image + registry: public-images + version: 3.10.0 + stacks: + - compute-plane + requirement: required + - name: nvcf-ai-api-gateway-service + type: image + registry: public-images + version: 1.35.1 + stacks: + - self-managed + requirement: optional + - name: nvcf-api-keys-service + type: image + registry: public-images + version: 1.9.1 + stacks: + - self-managed + requirement: required + - name: nvcf-cassandra-migrations + type: image + registry: public-images + version: 0.17.6 + stacks: + - self-managed + requirement: required + - name: nvcf-ess + type: image + registry: public-images + version: 0.5.0 + stacks: + - self-managed + requirement: required + - name: nvcf-function-autoscaler + type: image + registry: public-images + version: 1.21.8 + stacks: + - self-managed + requirement: required + - name: nvcf-grpc-proxy + type: image + registry: public-images + version: 1.33.5 + stacks: + - self-managed + requirement: required + - name: nvcf-image-credential-helper + type: image + registry: public-images + version: 0.11.1 + stacks: + - compute-plane + requirement: required + - name: nvcf-invocation-service + type: image + registry: public-images + version: 0.12.1 + stacks: + - self-managed + requirement: required + - name: nvcf-nats-auth-callout-service + type: image + registry: public-images + version: 0.8.3 + stacks: + - self-managed + requirement: required + - name: nvcf-notary + type: image + registry: public-images + version: 1.14.1 + stacks: + - self-managed + requirement: required + - name: nvcf-openbao-migrations + type: image + registry: public-images + version: 0.19.5 + stacks: + - self-managed + requirement: required + - name: nvcf-openbao + type: image + registry: public-images + version: 2.6.2-nv-1.3.4 + stacks: + - self-managed + requirement: required + - name: nvcf-ratelimiter + type: image + registry: public-images + version: 1.17.3 + stacks: + - self-managed + requirement: required + - name: nvcf-service-oss + type: image + registry: public-images + version: 1.18.0 + stacks: + - self-managed + requirement: required + - name: nvcf-state-metrics-service + type: image + registry: public-images + version: 1.23.7 + stacks: + - self-managed + requirement: required + - name: nvcf-ui + type: image + registry: public-images + version: 1.1.2 + stacks: + - self-managed + requirement: optional + - name: nvcf-worker-init-oss + type: image + registry: public-images + version: 1.2.1 + stacks: + - self-managed + requirement: required + - name: nvcf-worker-llm-credentials-oss + type: image + registry: public-images + version: 1.1.2 + stacks: + - self-managed + requirement: required + - name: nvcf-worker-utils-oss + type: image + registry: public-images + version: 1.2.3 + stacks: + - self-managed + requirement: required + - name: nvct-service-oss + type: image + registry: public-images + version: 1.66.0 + stacks: + - self-managed + requirement: required + - name: oss-vault-k8s + type: image + registry: public-images + version: 1.7.4 + stacks: + - self-managed + requirement: required + - name: reval-server + type: image + registry: public-images + version: 0.20.2 + stacks: + - self-managed + requirement: required + - name: stargate + type: image + registry: public-images + version: 0.18.0 + stacks: + - self-managed + requirement: optional + - name: cert-manager-acmesolver + type: image + registry: public-images + upstream_repository: quay.io/jetstack/cert-manager-acmesolver + version: v1.20.2 + stacks: + - self-managed + requirement: optional + - name: victoria-metrics + type: image + registry: public-images + upstream_repository: docker.io/victoriametrics/victoria-metrics + version: v1.150.0 + stacks: + - observability + requirement: required + - name: nvcf-cluster-topology + type: chart + registry: public-helm + version: 0.1.0 + stacks: + - compute-plane + requirement: required + - name: nvcf-default-monitors + type: chart + registry: public-helm + version: 0.2.0 + stacks: + - observability + requirement: required + - name: nvcf-otel-collector + type: chart + registry: public-helm + version: 0.2.0 + stacks: + - observability + requirement: required + - name: dynamo-platform + type: chart + registry: public-helm + upstream_repository: https://helm.ngc.nvidia.com/nvidia/ai-dynamo + version: 1.4.2 + stacks: + - compute-plane + requirement: optional + - name: helm-admin-token-issuer-proxy + type: chart + registry: public-helm + version: 1.5.3 + stacks: + - self-managed + requirement: required + - name: helm-nvca-operator + type: chart + registry: public-helm + version: 1.28.0 + stacks: + - compute-plane + requirement: required + - name: helm-nvcf-api-keys + type: chart + registry: public-helm + version: 1.8.0 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-api + type: chart + registry: public-helm + version: 1.27.1 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-cassandra + type: chart + registry: public-helm + version: 0.21.3 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-cert-manager + type: chart + registry: public-helm + version: 0.1.0 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-ess-api + type: chart + registry: public-helm + version: 1.8.2 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-function-autoscaler + type: chart + registry: public-helm + version: 0.5.2 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-grpc-proxy + type: chart + registry: public-helm + version: 1.7.4 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-invocation-service + type: chart + registry: public-helm + version: 1.6.1 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-llm-api-gateway + type: chart + registry: public-helm + version: 1.4.3 + stacks: + - self-managed + requirement: optional + - name: helm-nvcf-llm-request-router + type: chart + registry: public-helm + version: 1.14.1 + stacks: + - self-managed + requirement: optional + - name: helm-nvcf-nats-auth-callout-service + type: chart + registry: public-helm + version: 1.2.1 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-nats + type: chart + registry: public-helm + version: 0.8.4 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-notary-service + type: chart + registry: public-helm + version: 1.6.0 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-nvct-api + type: chart + registry: public-helm + version: 1.6.0 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-openbao-server + type: chart + registry: public-helm + version: 0.32.6 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-pki + type: chart + registry: public-helm + version: 0.1.0 + stacks: + - self-managed + requirement: optional + - name: helm-nvcf-rate-limiter + type: chart + registry: public-helm + version: 1.2.1 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-sis + type: chart + registry: public-helm + version: 2.4.0 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-state-metrics + type: chart + registry: public-helm + version: 1.0.2 + stacks: + - self-managed + requirement: required + - name: helm-nvcf-ui + type: chart + registry: public-helm + version: 1.1.2 + stacks: + - self-managed + requirement: optional + - name: helm-nvcf-vanity-gateway + type: chart + registry: public-helm + version: 0.5.0 + stacks: + - self-managed + requirement: optional + - name: helm-reval + type: chart + registry: public-helm + version: 1.4.1 + stacks: + - self-managed + requirement: required + - name: nvcf-gateway-routes + type: chart + registry: public-helm + version: 1.18.2 + stacks: + - self-managed + requirement: required + - id: opentelemetry-operator-chart + name: opentelemetry-operator + type: chart + registry: public-helm + upstream_repository: https://open-telemetry.github.io/opentelemetry-helm-charts + version: 0.122.0 + stacks: + - observability + requirement: required + - name: prometheus-operator-crds + type: chart + registry: public-helm + upstream_repository: https://prometheus-community.github.io/helm-charts + version: 31.0.1 + stacks: + - observability + requirement: required + - name: victoria-metrics-single + type: chart + registry: public-helm + upstream_repository: https://victoriametrics.github.io/helm-charts + version: 0.45.0 + stacks: + - observability + requirement: required + - name: grove-charts + type: chart + registry: public-helm + upstream_repository: oci://ghcr.io/ai-dynamo/grove + version: v0.1.0-alpha.12 + stacks: + - compute-plane + requirement: optional + - name: kai-scheduler + type: chart + registry: public-helm + upstream_repository: oci://ghcr.io/kai-scheduler/kai-scheduler + version: v0.17.1 + stacks: + - compute-plane + requirement: optional + - name: pylon + type: image + registry: public-images + version: 0.18.0 +supplemental_artifacts: + - name: nvcf-cli + type: resource + registry: public-resources + version: 1.16.2 + - name: nvcf-compute-plane-stack + type: resource + registry: public-resources + version: 1.0.0 + stacks: + - compute-plane + - name: nvcf-observability-stack + type: resource + registry: public-resources + version: 1.0.0 + stacks: + - observability + - name: nvcf-observability-reference-stack + type: chart + registry: public-helm + version: 1.10.0 + - name: nvcf-example-dashboards + type: chart + registry: public-helm + version: 1.6.0 + - id: nvcf-container-cache-image + name: nvcf-container-cache + type: image + registry: public-images + version: v1.1.36 + - id: nvcf-container-cache-chart + name: nvcf-container-cache + type: chart + registry: public-helm + version: 0.25.22 + - name: nvcf-proxy-tls-certs + type: image + registry: public-images + version: v1.2.10 +outputs: + - path: docs/overview/compatibility-matrix.md + blocks: + - marker: compatibility-matrix + renderer: compatibility-matrix + - path: docs/overview/manifest.md + blocks: + - marker: manifest-artifact-registry-paths + renderer: manifest-artifact-registry-paths + - path: docs/overview/image-mirroring.md + blocks: + - marker: image-mirroring-resource-examples + renderer: image-mirroring-resource-examples + - marker: image-mirroring-stack-snippet + renderer: image-mirroring-stack-snippet + - marker: image-mirroring-compute-stack-snippet + renderer: image-mirroring-compute-stack-snippet + - marker: image-mirroring-observability-stack-snippet + renderer: image-mirroring-observability-stack-snippet + - marker: image-mirroring-cli-snippet + renderer: image-mirroring-cli-snippet + - path: docs/compute-plane/cluster-management/self-managed.md + - path: docs/compute-plane/cluster-management/reference.md diff --git a/docs/version-catalog/cp-0.20.6-compute-0.4.4-obs-0.2.2.yaml b/docs/version-catalog/self-managed-1.0.yaml similarity index 100% rename from docs/version-catalog/cp-0.20.6-compute-0.4.4-obs-0.2.2.yaml rename to docs/version-catalog/self-managed-1.0.yaml diff --git a/examples/function-samples/helmchart-samples/dynamo-operator-sample/README.md b/examples/function-samples/helmchart-samples/dynamo-operator-sample/README.md index 4dc92fd10d..0014fe0479 100644 --- a/examples/function-samples/helmchart-samples/dynamo-operator-sample/README.md +++ b/examples/function-samples/helmchart-samples/dynamo-operator-sample/README.md @@ -4,9 +4,9 @@ This demo adapts an [upstream disaggregated router example](https://github.com/ai-dynamo/dynamo/blob/v1.0.2/examples/backends/vllm/deploy/disagg_router.yaml) for an NVCF Helm function. Dynamo uses Grove and KAI Scheduler to orchestrate and place the workload. See -[Gang Scheduling](../../../../docs/user/cluster-management/gang-scheduling.md) +[Gang Scheduling](../../../../docs/compute-plane/cluster-management/gang-scheduling.md) and -[Topology-Aware Scheduling](../../../../docs/user/cluster-management/topology-aware-scheduling.md) +[Topology-Aware Scheduling](../../../../docs/compute-plane/cluster-management/topology-aware-scheduling.md) for the production compute plane configuration. ## Prerequisites diff --git a/examples/function-samples/helmchart-samples/inference-test-sample/README.md b/examples/function-samples/helmchart-samples/inference-test-sample/README.md index 46c957d8ce..2e5961d51c 100644 --- a/examples/function-samples/helmchart-samples/inference-test-sample/README.md +++ b/examples/function-samples/helmchart-samples/inference-test-sample/README.md @@ -50,7 +50,7 @@ nvcf-cli registry add \ To opt into worker-readiness-based instance health, set `statusByWorkerReadiness: true` in `inference-test/values.yaml`. This renders the optional `nvcf-workload-config` ConfigMap described in -[Helm Functions](../../../../docs/user/helm-functions.md#use-worker-readiness-for-function-health). +[Helm Functions](../../../../docs/overview/helm-functions.md#use-worker-readiness-for-function-health). The chart declares CPU and memory requests and limits because self-managed NVCF validates resource limits before admitting Helm function workloads. diff --git a/examples/function-samples/helmchart-samples/inference-test-sample/inference-test/values.yaml b/examples/function-samples/helmchart-samples/inference-test-sample/inference-test/values.yaml index f4140e453b..6cd2b7f5ad 100644 --- a/examples/function-samples/helmchart-samples/inference-test-sample/inference-test/values.yaml +++ b/examples/function-samples/helmchart-samples/inference-test-sample/inference-test/values.yaml @@ -31,5 +31,5 @@ resources: memory: 1Gi # When true, render nvcf-workload-config so NVCA uses worker container readiness -# for instance health instead of all chart objects. See docs/user/helm-functions.md. +# for instance health instead of all chart objects. See docs/overview/helm-functions.md. statusByWorkerReadiness: false diff --git a/fern/docs.yml b/fern/docs.yml index f7358b3962..fdefc0e6a5 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -20,30 +20,63 @@ logo: href: /nvcf/overview right-text: Cloud Functions - navbar-links: - type: github value: https://github.com/NVIDIA/nvcf -versions: -- display-name: "Latest (cp-0.20.6-compute-0.4.4-obs-0.2.2)" - path: versions/cp-0.20.6-compute-0.4.4-obs-0.2.2.yml - slug: "" -- display-name: dev - path: versions/dev.yml - slug: dev -- display-name: "cp-0.20.6-compute-0.4.4-obs-0.2.2 (CP 0.20.6, Compute 0.4.4, Obs 0.2.2)" - path: versions/cp-0.20.6-compute-0.4.4-obs-0.2.2.yml - slug: "cp-0.20.6-compute-0.4.4-obs-0.2.2" -- display-name: "0.6.1" - path: versions/v0.6.1.yml - slug: "v0.6.1" -- display-name: "0.6.0" - path: versions/v0.6.0.yml - slug: "v0.6.0" -- display-name: "0.5.0" - path: versions/v0.5.yml - slug: "v0.5" +# Each stack is a product with its own version menu. Overview is unversioned +# and holds the compatibility matrix and shared guides. Legacy full-tree +# versions predate the per-stack split and are attached to Self-Managed Stack. +products: + - display-name: Overview + subtitle: Compatibility matrix and shared guides + slug: overview + path: products/overview.yml + + - display-name: Self-Managed Stack + subtitle: Control plane installation and operations + slug: self-managed + path: products/self-managed/1.0.yml + versions: + - display-name: "1.0" + path: products/self-managed/1.0.yml + slug: "1.0" + - display-name: dev + path: products/self-managed/dev.yml + slug: dev + - display-name: "0.6.1 (legacy)" + path: products/self-managed/v0.6.1.yml + slug: "v0.6.1" + - display-name: "0.6.0 (legacy)" + path: products/self-managed/v0.6.0.yml + slug: "v0.6.0" + - display-name: "0.5.0 (legacy)" + path: products/self-managed/v0.5.yml + slug: "v0.5" + + - display-name: Compute Plane Stack + subtitle: GPU cluster setup and scheduling + slug: compute-plane + path: products/compute-plane/1.0.yml + versions: + - display-name: "1.0" + path: products/compute-plane/1.0.yml + slug: "1.0" + - display-name: dev + path: products/compute-plane/dev.yml + slug: dev + + - display-name: Observability Stack + subtitle: Metrics, dashboards, and alerting + slug: observability + path: products/observability/1.0.yml + versions: + - display-name: "1.0" + path: products/observability/1.0.yml + slug: "1.0" + - display-name: dev + path: products/observability/dev.yml + slug: dev check: rules: @@ -51,54 +84,592 @@ check: redirects: - source: "/nvcf/index.html" - destination: "/nvcf/" - - source: "/nvcf/dev/index.html" - destination: "/nvcf/dev/" + destination: "/nvcf/overview" - source: "/nvcf/latest/index.html" - destination: "/nvcf/" - - source: "/nvcf/v0.6.0-rc/index.html" - destination: "/nvcf/v0.6.0/" - - source: "/nvcf/v0.6.0/index.html" - destination: "/nvcf/v0.6.0/" - - source: "/nvcf/v0.6.1-rc/index.html" - destination: "/nvcf/v0.6.1/" - - source: "/nvcf/v0.6.1/index.html" - destination: "/nvcf/v0.6.1/" - - source: "/nvcf/v0.5/index.html" - destination: "/nvcf/v0.5/" - - source: "/nvcf/dev/0-6-1-release-notes" - destination: "/nvcf/v0.6.1/0-6-1-release-notes" - - source: "/nvcf/dev/0-6-0-to-0-6-1-upgrade" - destination: "/nvcf/v0.6.1/0-6-0-to-0-6-1-upgrade" - - source: "/nvcf/dev/0-6-0-release-notes" - destination: "/nvcf/v0.6.0/0-6-0-release-notes" - - source: "/nvcf/dev/0-5-0-to-0-6-0-upgrade" - destination: "/nvcf/v0.6.0/0-5-0-to-0-6-0-upgrade" - - source: "/nvcf/dev/function-autoscaling-overview" - destination: "/nvcf/function-autoscaling-overview" + destination: "/nvcf/overview" + - source: "/nvcf/dev/index.html" + destination: "/nvcf/overview" + - source: "/nvcf/v0.6.0-rc/:slug*" + destination: "/nvcf/self-managed/v0.6.0/:slug*" + - source: "/nvcf/v0.6.0/:slug*" + destination: "/nvcf/self-managed/v0.6.0/:slug*" + - source: "/nvcf/v0.6.1-rc/:slug*" + destination: "/nvcf/self-managed/v0.6.1/:slug*" + - source: "/nvcf/v0.6.1/:slug*" + destination: "/nvcf/self-managed/v0.6.1/:slug*" + - source: "/nvcf/v0.5/:slug*" + destination: "/nvcf/self-managed/v0.5/:slug*" + - source: "/nvcf/cp-0.20.6-compute-0.4.4-obs-0.2.2/:slug*" + destination: "/nvcf/self-managed/1.0/:slug*" + - source: "/nvcf/self-managed/cp-0.20.6-compute-0.4.4-obs-0.2.2/:slug*" + destination: "/nvcf/self-managed/1.0/:slug*" + # Pages moved from the flat dev tree into products. Old default-version + # URLs (/nvcf/) and dev URLs (/nvcf/dev/) both redirect. + - source: "/nvcf/overview" + destination: "/nvcf/overview/overview" + - source: "/nvcf/dev/overview" + destination: "/nvcf/overview/overview" + - source: "/nvcf/compatibility-matrix" + destination: "/nvcf/overview/compatibility-matrix" + - source: "/nvcf/dev/compatibility-matrix" + destination: "/nvcf/overview/compatibility-matrix" + - source: "/nvcf/quickstart" + destination: "/nvcf/overview/quickstart" + - source: "/nvcf/dev/quickstart" + destination: "/nvcf/overview/quickstart" + - source: "/nvcf/release-notes-overview" + destination: "/nvcf/overview/release-notes-overview" + - source: "/nvcf/dev/release-notes-overview" + destination: "/nvcf/overview/release-notes-overview" + - source: "/nvcf/infrastructure-sizing" + destination: "/nvcf/overview/infrastructure-sizing" + - source: "/nvcf/dev/infrastructure-sizing" + destination: "/nvcf/overview/infrastructure-sizing" + - source: "/nvcf/manifest" + destination: "/nvcf/overview/manifest" + - source: "/nvcf/dev/manifest" + destination: "/nvcf/overview/manifest" + - source: "/nvcf/image-mirroring" + destination: "/nvcf/overview/image-mirroring" + - source: "/nvcf/dev/image-mirroring" + destination: "/nvcf/overview/image-mirroring" + - source: "/nvcf/architecture-overview" + destination: "/nvcf/overview/architecture-overview" + - source: "/nvcf/dev/architecture-overview" + destination: "/nvcf/overview/architecture-overview" + - source: "/nvcf/local-development" + destination: "/nvcf/overview/local-development" + - source: "/nvcf/dev/local-development" + destination: "/nvcf/overview/local-development" + - source: "/nvcf/single-cluster-cli" + destination: "/nvcf/overview/single-cluster-cli" + - source: "/nvcf/dev/single-cluster-cli" + destination: "/nvcf/overview/single-cluster-cli" + - source: "/nvcf/single-cluster-helmfile" + destination: "/nvcf/overview/single-cluster-helmfile" + - source: "/nvcf/dev/single-cluster-helmfile" + destination: "/nvcf/overview/single-cluster-helmfile" + - source: "/nvcf/multi-cluster-cli" + destination: "/nvcf/overview/multi-cluster-cli" + - source: "/nvcf/dev/multi-cluster-cli" + destination: "/nvcf/overview/multi-cluster-cli" + - source: "/nvcf/multi-cluster-helmfile" + destination: "/nvcf/overview/multi-cluster-helmfile" + - source: "/nvcf/dev/multi-cluster-helmfile" + destination: "/nvcf/overview/multi-cluster-helmfile" + - source: "/nvcf/ngc-managed-clusters" + destination: "/nvcf/overview/ngc-managed-clusters" + - source: "/nvcf/dev/ngc-managed-clusters" + destination: "/nvcf/overview/ngc-managed-clusters" + - source: "/nvcf/helm-managed-clusters" + destination: "/nvcf/overview/helm-managed-clusters" + - source: "/nvcf/dev/helm-managed-clusters" + destination: "/nvcf/overview/helm-managed-clusters" + - source: "/nvcf/git-ops-cluster-registration" + destination: "/nvcf/overview/git-ops-cluster-registration" + - source: "/nvcf/dev/git-ops-cluster-registration" + destination: "/nvcf/overview/git-ops-cluster-registration" + - source: "/nvcf/configuration" + destination: "/nvcf/overview/configuration" + - source: "/nvcf/dev/configuration" + destination: "/nvcf/overview/configuration" + - source: "/nvcf/monitoring" + destination: "/nvcf/overview/monitoring" + - source: "/nvcf/dev/monitoring" + destination: "/nvcf/overview/monitoring" + - source: "/nvcf/reference" + destination: "/nvcf/overview/reference" + - source: "/nvcf/dev/reference" + destination: "/nvcf/overview/reference" + - source: "/nvcf/function-lifecycle" + destination: "/nvcf/overview/function-lifecycle" - source: "/nvcf/dev/function-lifecycle" - destination: "/nvcf/function-lifecycle" + destination: "/nvcf/overview/function-lifecycle" + - source: "/nvcf/service-keys" + destination: "/nvcf/overview/service-keys" + - source: "/nvcf/dev/service-keys" + destination: "/nvcf/overview/service-keys" + - source: "/nvcf/managed-observability" + destination: "/nvcf/overview/managed-observability" + - source: "/nvcf/dev/managed-observability" + destination: "/nvcf/overview/managed-observability" + - source: "/nvcf/installation-overview" + destination: "/nvcf/self-managed/installation-overview" + - source: "/nvcf/dev/installation-overview" + destination: "/nvcf/self-managed/installation-overview" + - source: "/nvcf/helmfile-installation" + destination: "/nvcf/self-managed/helmfile-installation" + - source: "/nvcf/dev/helmfile-installation" + destination: "/nvcf/self-managed/helmfile-installation" + - source: "/nvcf/csp-end-to-end-example" + destination: "/nvcf/self-managed/csp-end-to-end-example" + - source: "/nvcf/dev/csp-end-to-end-example" + destination: "/nvcf/self-managed/csp-end-to-end-example" + - source: "/nvcf/nvcf-ui" + destination: "/nvcf/self-managed/nvcf-ui" + - source: "/nvcf/dev/nvcf-ui" + destination: "/nvcf/self-managed/nvcf-ui" + - source: "/nvcf/optional-enhancements" + destination: "/nvcf/self-managed/gateway-routing" + - source: "/nvcf/dev/optional-enhancements" + destination: "/nvcf/self-managed/gateway-routing" + - source: "/nvcf/self-managed/optional-enhancements" + destination: "/nvcf/self-managed/gateway-routing" + - source: "/nvcf/pod-disruption-budgets" + destination: "/nvcf/self-managed/pod-disruption-budgets" + - source: "/nvcf/dev/pod-disruption-budgets" + destination: "/nvcf/self-managed/pod-disruption-budgets" + - source: "/nvcf/g-rpc-invocation-enablement" + destination: "/nvcf/self-managed/g-rpc-invocation-enablement" + - source: "/nvcf/dev/g-rpc-invocation-enablement" + destination: "/nvcf/self-managed/g-rpc-invocation-enablement" + - source: "/nvcf/llm-function-enablement" + destination: "/nvcf/self-managed/llm-function-enablement" + - source: "/nvcf/dev/llm-function-enablement" + destination: "/nvcf/self-managed/llm-function-enablement" + - source: "/nvcf/llm-request-router-load-balancing" + destination: "/nvcf/self-managed/llm-request-router-load-balancing" + - source: "/nvcf/dev/llm-request-router-load-balancing" + destination: "/nvcf/self-managed/llm-request-router-load-balancing" + - source: "/nvcf/gateway-routing" + destination: "/nvcf/self-managed/gateway-routing" + - source: "/nvcf/dev/gateway-routing" + destination: "/nvcf/self-managed/gateway-routing" + - source: "/nvcf/third-party-registries" + destination: "/nvcf/self-managed/third-party-registries" + - source: "/nvcf/dev/third-party-registries" + destination: "/nvcf/self-managed/third-party-registries" + - source: "/nvcf/registry-allowlist" + destination: "/nvcf/self-managed/registry-allowlist" + - source: "/nvcf/dev/registry-allowlist" + destination: "/nvcf/self-managed/registry-allowlist" + - source: "/nvcf/lls-installation" + destination: "/nvcf/self-managed/lls-installation" + - source: "/nvcf/dev/lls-installation" + destination: "/nvcf/self-managed/lls-installation" + - source: "/nvcf/api" + destination: "/nvcf/overview/api" + - source: "/nvcf/dev/api" + destination: "/nvcf/overview/api" + - source: "/nvcf/self-managed/api" + destination: "/nvcf/overview/api" + - source: "/nvcf/function-creation" + destination: "/nvcf/overview/function-creation" + - source: "/nvcf/dev/function-creation" + destination: "/nvcf/overview/function-creation" + - source: "/nvcf/self-managed/function-creation" + destination: "/nvcf/overview/function-creation" + - source: "/nvcf/llm-gateway" + destination: "/nvcf/overview/llm-gateway" + - source: "/nvcf/dev/llm-gateway" + destination: "/nvcf/overview/llm-gateway" + - source: "/nvcf/self-managed/llm-gateway" + destination: "/nvcf/overview/llm-gateway" + - source: "/nvcf/generic-http-function-invocation" + destination: "/nvcf/overview/generic-http-function-invocation" - source: "/nvcf/dev/generic-http-function-invocation" - destination: "/nvcf/generic-http-function-invocation" + destination: "/nvcf/overview/generic-http-function-invocation" + - source: "/nvcf/self-managed/generic-http-function-invocation" + destination: "/nvcf/overview/generic-http-function-invocation" + - source: "/nvcf/g-rpc-function-invocation" + destination: "/nvcf/overview/g-rpc-function-invocation" + - source: "/nvcf/dev/g-rpc-function-invocation" + destination: "/nvcf/overview/g-rpc-function-invocation" + - source: "/nvcf/self-managed/g-rpc-function-invocation" + destination: "/nvcf/overview/g-rpc-function-invocation" + - source: "/nvcf/container-functions" + destination: "/nvcf/overview/container-functions" + - source: "/nvcf/dev/container-functions" + destination: "/nvcf/overview/container-functions" + - source: "/nvcf/self-managed/container-functions" + destination: "/nvcf/overview/container-functions" + - source: "/nvcf/helm-functions" + destination: "/nvcf/overview/helm-functions" + - source: "/nvcf/dev/helm-functions" + destination: "/nvcf/overview/helm-functions" + - source: "/nvcf/self-managed/helm-functions" + destination: "/nvcf/overview/helm-functions" + - source: "/nvcf/streaming-functions" + destination: "/nvcf/overview/streaming-functions" + - source: "/nvcf/dev/streaming-functions" + destination: "/nvcf/overview/streaming-functions" + - source: "/nvcf/self-managed/streaming-functions" + destination: "/nvcf/overview/streaming-functions" + - source: "/nvcf/configure-autoscaling" + destination: "/nvcf/overview/configure-autoscaling" + - source: "/nvcf/dev/configure-autoscaling" + destination: "/nvcf/overview/configure-autoscaling" + - source: "/nvcf/self-managed/configure-autoscaling" + destination: "/nvcf/overview/configure-autoscaling" + - source: "/nvcf/cli" + destination: "/nvcf/overview/cli" + - source: "/nvcf/dev/cli" + destination: "/nvcf/overview/cli" + - source: "/nvcf/self-managed/cli" + destination: "/nvcf/overview/cli" + - source: "/nvcf/task-creation" + destination: "/nvcf/overview/task-creation" + - source: "/nvcf/dev/task-creation" + destination: "/nvcf/overview/task-creation" + - source: "/nvcf/self-managed/task-creation" + destination: "/nvcf/overview/task-creation" + - source: "/nvcf/container-based-task-creation" + destination: "/nvcf/overview/container-based-task-creation" + - source: "/nvcf/dev/container-based-task-creation" + destination: "/nvcf/overview/container-based-task-creation" + - source: "/nvcf/self-managed/container-based-task-creation" + destination: "/nvcf/overview/container-based-task-creation" + - source: "/nvcf/helm-based-task-creation" + destination: "/nvcf/overview/helm-based-task-creation" + - source: "/nvcf/dev/helm-based-task-creation" + destination: "/nvcf/overview/helm-based-task-creation" + - source: "/nvcf/self-managed/helm-based-task-creation" + destination: "/nvcf/overview/helm-based-task-creation" + - source: "/nvcf/function-autoscaling-overview" + destination: "/nvcf/self-managed/function-autoscaling-overview" + - source: "/nvcf/dev/function-autoscaling-overview" + destination: "/nvcf/self-managed/function-autoscaling-overview" + - source: "/nvcf/architecture" + destination: "/nvcf/self-managed/architecture" + - source: "/nvcf/dev/architecture" + destination: "/nvcf/self-managed/architecture" + - source: "/nvcf/operations" + destination: "/nvcf/self-managed/operations" + - source: "/nvcf/dev/operations" + destination: "/nvcf/self-managed/operations" + - source: "/nvcf/autoscaler-observability" + destination: "/nvcf/observability/autoscaler" + - source: "/nvcf/dev/autoscaler-observability" + destination: "/nvcf/observability/autoscaler" + - source: "/nvcf/self-managed/autoscaler-observability" + destination: "/nvcf/observability/autoscaler" + - source: "/nvcf/control-plane-operations" + destination: "/nvcf/self-managed/control-plane-operations" + - source: "/nvcf/dev/control-plane-operations" + destination: "/nvcf/self-managed/control-plane-operations" + - source: "/nvcf/troubleshooting" + destination: "/nvcf/self-managed/troubleshooting" + - source: "/nvcf/dev/troubleshooting" + destination: "/nvcf/self-managed/troubleshooting" + - source: "/nvcf/runbooks" + destination: "/nvcf/self-managed/control-plane-operations" + - source: "/nvcf/dev/runbooks" + destination: "/nvcf/self-managed/control-plane-operations" + - source: "/nvcf/self-managed/runbooks" + destination: "/nvcf/self-managed/control-plane-operations" + - source: "/nvcf/key-rotation" + destination: "/nvcf/self-managed/key-rotation" + - source: "/nvcf/dev/key-rotation" + destination: "/nvcf/self-managed/key-rotation" + - source: "/nvcf/transport-tls-rotation" + destination: "/nvcf/self-managed/transport-tls-rotation" + - source: "/nvcf/dev/transport-tls-rotation" + destination: "/nvcf/self-managed/transport-tls-rotation" + - source: "/nvcf/caches-overview" + destination: "/nvcf/compute-plane/caches-overview" + - source: "/nvcf/dev/caches-overview" + destination: "/nvcf/compute-plane/caches-overview" + - source: "/nvcf/self-managed/caches-overview" + destination: "/nvcf/compute-plane/caches-overview" + - source: "/nvcf/quota-management" + destination: "/nvcf/self-managed/quota-management" + - source: "/nvcf/dev/quota-management" + destination: "/nvcf/self-managed/quota-management" + - source: "/nvcf/g-rpc-load-testing" + destination: "/nvcf/overview/g-rpc-load-testing" + - source: "/nvcf/dev/g-rpc-load-testing" + destination: "/nvcf/overview/g-rpc-load-testing" + - source: "/nvcf/self-managed/g-rpc-load-testing" + destination: "/nvcf/overview/g-rpc-load-testing" + - source: "/nvcf/g-rpc-load-test-sli-guide" + destination: "/nvcf/overview/g-rpc-load-test-sli-guide" - source: "/nvcf/dev/g-rpc-load-test-sli-guide" - destination: "/nvcf/g-rpc-load-test-sli-guide" + destination: "/nvcf/overview/g-rpc-load-test-sli-guide" + - source: "/nvcf/self-managed/g-rpc-load-test-sli-guide" + destination: "/nvcf/overview/g-rpc-load-test-sli-guide" + - source: "/nvcf/http-load-testing" + destination: "/nvcf/overview/http-load-testing" - source: "/nvcf/dev/http-load-testing" - destination: "/nvcf/http-load-testing" + destination: "/nvcf/overview/http-load-testing" + - source: "/nvcf/self-managed/http-load-testing" + destination: "/nvcf/overview/http-load-testing" + - source: "/nvcf/http-load-test-sli-guide" + destination: "/nvcf/overview/http-load-test-sli-guide" - source: "/nvcf/dev/http-load-test-sli-guide" - destination: "/nvcf/http-load-test-sli-guide" + destination: "/nvcf/overview/http-load-test-sli-guide" + - source: "/nvcf/self-managed/http-load-test-sli-guide" + destination: "/nvcf/overview/http-load-test-sli-guide" + - source: "/nvcf/http-soak-testing" + destination: "/nvcf/overview/http-soak-testing" + - source: "/nvcf/dev/http-soak-testing" + destination: "/nvcf/overview/http-soak-testing" + - source: "/nvcf/self-managed/http-soak-testing" + destination: "/nvcf/overview/http-soak-testing" + - source: "/nvcf/gpu-cluster-setup" + destination: "/nvcf/compute-plane/gpu-cluster-setup" + - source: "/nvcf/dev/gpu-cluster-setup" + destination: "/nvcf/compute-plane/gpu-cluster-setup" + - source: "/nvcf/self-managed-clusters" + destination: "/nvcf/compute-plane/self-managed-clusters" + - source: "/nvcf/dev/self-managed-clusters" + destination: "/nvcf/compute-plane/self-managed-clusters" + - source: "/nvcf/nsight-profiling" + destination: "/nvcf/compute-plane/nsight-profiling" + - source: "/nvcf/dev/nsight-profiling" + destination: "/nvcf/compute-plane/nsight-profiling" + - source: "/nvcf/fake-gpu-operator" + destination: "/nvcf/compute-plane/fake-gpu-operator" + - source: "/nvcf/dev/fake-gpu-operator" + destination: "/nvcf/compute-plane/fake-gpu-operator" + - source: "/nvcf/cluster-configuration" + destination: "/nvcf/compute-plane/cluster-configuration" + - source: "/nvcf/dev/cluster-configuration" + destination: "/nvcf/compute-plane/cluster-configuration" + - source: "/nvcf/multi-tenancy" + destination: "/nvcf/overview/multi-tenancy" + - source: "/nvcf/dev/multi-tenancy" + destination: "/nvcf/overview/multi-tenancy" + - source: "/nvcf/compute-plane/multi-tenancy" + destination: "/nvcf/overview/multi-tenancy" + - source: "/nvcf/kai-scheduler" + destination: "/nvcf/compute-plane/kai-scheduler" + - source: "/nvcf/dev/kai-scheduler" + destination: "/nvcf/compute-plane/kai-scheduler" + - source: "/nvcf/gang-scheduling" + destination: "/nvcf/compute-plane/gang-scheduling" + - source: "/nvcf/dev/gang-scheduling" + destination: "/nvcf/compute-plane/gang-scheduling" + - source: "/nvcf/topology-aware-scheduling" + destination: "/nvcf/compute-plane/topology-aware-scheduling" + - source: "/nvcf/dev/topology-aware-scheduling" + destination: "/nvcf/compute-plane/topology-aware-scheduling" + - source: "/nvcf/helm-model-caching" + destination: "/nvcf/compute-plane/helm-model-caching" + - source: "/nvcf/dev/helm-model-caching" + destination: "/nvcf/compute-plane/helm-model-caching" + - source: "/nvcf/container-cache" + destination: "/nvcf/compute-plane/container-cache" + - source: "/nvcf/dev/container-cache" + destination: "/nvcf/compute-plane/container-cache" + - source: "/nvcf/gx-cache" + destination: "/nvcf/compute-plane/gx-cache" + - source: "/nvcf/dev/gx-cache" + destination: "/nvcf/compute-plane/gx-cache" + - source: "/nvcf/simulation-caches" + destination: "/nvcf/compute-plane/simulation-caches" + - source: "/nvcf/dev/simulation-caches" + destination: "/nvcf/compute-plane/simulation-caches" + - source: "/nvcf/cluster-monitoring" + destination: "/nvcf/compute-plane/cluster-monitoring" + - source: "/nvcf/dev/cluster-monitoring" + destination: "/nvcf/compute-plane/cluster-monitoring" + - source: "/nvcf/cluster-reference" + destination: "/nvcf/compute-plane/cluster-reference" + - source: "/nvcf/dev/cluster-reference" + destination: "/nvcf/compute-plane/cluster-reference" + - source: "/nvcf/observability" + destination: "/nvcf/observability/observability" + - source: "/nvcf/dev/observability" + destination: "/nvcf/observability/observability" + - source: "/nvcf/example-dashboards" + destination: "/nvcf/observability/example-dashboards" + - source: "/nvcf/dev/example-dashboards" + destination: "/nvcf/observability/example-dashboards" + - source: "/nvcf/metrics-overview" + destination: "/nvcf/observability/metrics-overview" + - source: "/nvcf/dev/metrics-overview" + destination: "/nvcf/observability/metrics-overview" + - source: "/nvcf/cassandra" + destination: "/nvcf/observability/cassandra" + - source: "/nvcf/dev/cassandra" + destination: "/nvcf/observability/cassandra" + - source: "/nvcf/ess" + destination: "/nvcf/observability/ess" + - source: "/nvcf/dev/ess" + destination: "/nvcf/observability/ess" + - source: "/nvcf/init-container" + destination: "/nvcf/observability/init-container" + - source: "/nvcf/dev/init-container" + destination: "/nvcf/observability/init-container" + - source: "/nvcf/invocation-service" + destination: "/nvcf/observability/invocation-service" + - source: "/nvcf/dev/invocation-service" + destination: "/nvcf/observability/invocation-service" + - source: "/nvcf/llm-api-gateway" + destination: "/nvcf/observability/llm-api-gateway" - source: "/nvcf/dev/llm-api-gateway" - destination: "/nvcf/llm-api-gateway" + destination: "/nvcf/observability/llm-api-gateway" + - source: "/nvcf/llm-function-invocation-metrics-report" + destination: "/nvcf/observability/llm-function-invocation-metrics-report" - source: "/nvcf/dev/llm-function-invocation-metrics-report" - destination: "/nvcf/llm-function-invocation-metrics-report" + destination: "/nvcf/observability/llm-function-invocation-metrics-report" + - source: "/nvcf/llm-request-router" + destination: "/nvcf/observability/llm-request-router" - source: "/nvcf/dev/llm-request-router" - destination: "/nvcf/llm-request-router" - - source: "/nvcf/dev/managed-observability" - destination: "/nvcf/managed-observability" - - source: "/nvcf/dev/monitoring" - destination: "/nvcf/monitoring" - - source: "/nvcf/dev/operations" - destination: "/nvcf/operations" - - source: "/nvcf/dev/registry-allowlist" - destination: "/nvcf/registry-allowlist" - - source: "/nvcf/cp-0.20.6-compute-0.4.4-obs-0.2.2/index.html" - destination: "/nvcf/cp-0.20.6-compute-0.4.4-obs-0.2.2/" + destination: "/nvcf/observability/llm-request-router" + - source: "/nvcf/nvcf-api" + destination: "/nvcf/observability/nvcf-api" + - source: "/nvcf/dev/nvcf-api" + destination: "/nvcf/observability/nvcf-api" + - source: "/nvcf/sis-spot" + destination: "/nvcf/observability/sis-spot" + - source: "/nvcf/dev/sis-spot" + destination: "/nvcf/observability/sis-spot" + - source: "/nvcf/state-metrics" + destination: "/nvcf/observability/state-metrics" + - source: "/nvcf/dev/state-metrics" + destination: "/nvcf/observability/state-metrics" + - source: "/nvcf/utils-container" + destination: "/nvcf/observability/utils-container" + - source: "/nvcf/dev/utils-container" + destination: "/nvcf/observability/utils-container" + - source: "/nvcf/vault-open-bao" + destination: "/nvcf/observability/vault-open-bao" + - source: "/nvcf/dev/vault-open-bao" + destination: "/nvcf/observability/vault-open-bao" + - source: "/nvcf/self-managed/1.0/api" + destination: "/nvcf/overview/api" + - source: "/nvcf/self-managed/1.0/function-creation" + destination: "/nvcf/overview/function-creation" + - source: "/nvcf/self-managed/1.0/llm-gateway" + destination: "/nvcf/overview/llm-gateway" + - source: "/nvcf/self-managed/1.0/generic-http-function-invocation" + destination: "/nvcf/overview/generic-http-function-invocation" + - source: "/nvcf/self-managed/1.0/g-rpc-function-invocation" + destination: "/nvcf/overview/g-rpc-function-invocation" + - source: "/nvcf/self-managed/1.0/container-functions" + destination: "/nvcf/overview/container-functions" + - source: "/nvcf/self-managed/1.0/helm-functions" + destination: "/nvcf/overview/helm-functions" + - source: "/nvcf/self-managed/1.0/streaming-functions" + destination: "/nvcf/overview/streaming-functions" + - source: "/nvcf/self-managed/1.0/configure-autoscaling" + destination: "/nvcf/overview/configure-autoscaling" + - source: "/nvcf/self-managed/1.0/cli" + destination: "/nvcf/overview/cli" + - source: "/nvcf/self-managed/1.0/task-creation" + destination: "/nvcf/overview/task-creation" + - source: "/nvcf/self-managed/1.0/container-based-task-creation" + destination: "/nvcf/overview/container-based-task-creation" + - source: "/nvcf/self-managed/1.0/helm-based-task-creation" + destination: "/nvcf/overview/helm-based-task-creation" + - source: "/nvcf/self-managed/1.0/g-rpc-load-testing" + destination: "/nvcf/overview/g-rpc-load-testing" + - source: "/nvcf/self-managed/1.0/g-rpc-load-test-sli-guide" + destination: "/nvcf/overview/g-rpc-load-test-sli-guide" + - source: "/nvcf/self-managed/1.0/http-load-testing" + destination: "/nvcf/overview/http-load-testing" + - source: "/nvcf/self-managed/1.0/http-load-test-sli-guide" + destination: "/nvcf/overview/http-load-test-sli-guide" + - source: "/nvcf/self-managed/1.0/http-soak-testing" + destination: "/nvcf/overview/http-soak-testing" + - source: "/nvcf/self-managed/1.0/architecture-overview" + destination: "/nvcf/overview/architecture-overview" + - source: "/nvcf/self-managed/1.0/caches-overview" + destination: "/nvcf/compute-plane/1.0/caches-overview" + - source: "/nvcf/self-managed/1.0/cassandra" + destination: "/nvcf/observability/1.0/cassandra" + - source: "/nvcf/self-managed/1.0/cluster-configuration" + destination: "/nvcf/compute-plane/1.0/cluster-configuration" + - source: "/nvcf/self-managed/1.0/cluster-monitoring" + destination: "/nvcf/compute-plane/1.0/cluster-monitoring" + - source: "/nvcf/self-managed/1.0/cluster-reference" + destination: "/nvcf/compute-plane/1.0/cluster-reference" + - source: "/nvcf/self-managed/1.0/configuration" + destination: "/nvcf/overview/configuration" + - source: "/nvcf/self-managed/1.0/container-cache" + destination: "/nvcf/overview/container-cache" + - source: "/nvcf/self-managed/1.0/ess" + destination: "/nvcf/observability/1.0/ess" + - source: "/nvcf/self-managed/1.0/example-dashboards" + destination: "/nvcf/observability/1.0/example-dashboards" + - source: "/nvcf/self-managed/1.0/fake-gpu-operator" + destination: "/nvcf/compute-plane/1.0/fake-gpu-operator" + - source: "/nvcf/self-managed/1.0/function-lifecycle" + destination: "/nvcf/overview/function-lifecycle" + - source: "/nvcf/self-managed/1.0/gpu-cluster-setup" + destination: "/nvcf/compute-plane/1.0/gpu-cluster-setup" + - source: "/nvcf/self-managed/1.0/gx-cache" + destination: "/nvcf/overview/gx-cache" + - source: "/nvcf/self-managed/1.0/gang-scheduling" + destination: "/nvcf/compute-plane/1.0/gang-scheduling" + - source: "/nvcf/self-managed/1.0/git-ops-cluster-registration" + destination: "/nvcf/overview/git-ops-cluster-registration" + - source: "/nvcf/self-managed/1.0/helm-model-caching" + destination: "/nvcf/compute-plane/1.0/helm-model-caching" + - source: "/nvcf/self-managed/1.0/helm-managed-clusters" + destination: "/nvcf/overview/helm-managed-clusters" + - source: "/nvcf/self-managed/1.0/image-mirroring" + destination: "/nvcf/overview/image-mirroring" + - source: "/nvcf/self-managed/1.0/infrastructure-sizing" + destination: "/nvcf/overview/infrastructure-sizing" + - source: "/nvcf/self-managed/1.0/init-container" + destination: "/nvcf/observability/1.0/init-container" + - source: "/nvcf/self-managed/1.0/invocation-service" + destination: "/nvcf/observability/1.0/invocation-service" + - source: "/nvcf/self-managed/1.0/kai-scheduler" + destination: "/nvcf/overview/kai-scheduler" + - source: "/nvcf/self-managed/1.0/llm-api-gateway" + destination: "/nvcf/observability/1.0/llm-api-gateway" + - source: "/nvcf/self-managed/1.0/llm-function-invocation-metrics-report" + destination: "/nvcf/observability/1.0/llm-function-invocation-metrics-report" + - source: "/nvcf/self-managed/1.0/llm-request-router" + destination: "/nvcf/observability/1.0/llm-request-router" + - source: "/nvcf/self-managed/1.0/local-development" + destination: "/nvcf/overview/local-development" + - source: "/nvcf/self-managed/1.0/manifest" + destination: "/nvcf/overview/manifest" + - source: "/nvcf/self-managed/1.0/metrics-overview" + destination: "/nvcf/observability/1.0/metrics-overview" + - source: "/nvcf/self-managed/1.0/monitoring" + destination: "/nvcf/overview/monitoring" + - source: "/nvcf/self-managed/1.0/multi-tenancy" + destination: "/nvcf/overview/multi-tenancy" + - source: "/nvcf/self-managed/1.0/multi-cluster-cli" + destination: "/nvcf/overview/multi-cluster-cli" + - source: "/nvcf/self-managed/1.0/multi-cluster-helmfile" + destination: "/nvcf/overview/multi-cluster-helmfile" + - source: "/nvcf/self-managed/1.0/ngc-managed-clusters" + destination: "/nvcf/overview/ngc-managed-clusters" + - source: "/nvcf/self-managed/1.0/nvcf-api" + destination: "/nvcf/observability/1.0/nvcf-api" + - source: "/nvcf/self-managed/1.0/nsight-profiling" + destination: "/nvcf/compute-plane/1.0/nsight-profiling" + - source: "/nvcf/self-managed/1.0/observability" + destination: "/nvcf/observability/1.0/observability" + - source: "/nvcf/self-managed/1.0/overview" + destination: "/nvcf/overview/overview" + - source: "/nvcf/self-managed/1.0/quickstart" + destination: "/nvcf/overview/quickstart" + - source: "/nvcf/self-managed/1.0/reference" + destination: "/nvcf/overview/reference" + - source: "/nvcf/self-managed/1.0/sis-spot" + destination: "/nvcf/observability/1.0/sis-spot" + - source: "/nvcf/self-managed/1.0/self-managed-clusters" + destination: "/nvcf/compute-plane/1.0/self-managed-clusters" + - source: "/nvcf/self-managed/1.0/service-keys" + destination: "/nvcf/overview/service-keys" + - source: "/nvcf/self-managed/1.0/simulation-caches" + destination: "/nvcf/compute-plane/1.0/simulation-caches" + - source: "/nvcf/self-managed/1.0/single-cluster-cli" + destination: "/nvcf/overview/single-cluster-cli" + - source: "/nvcf/self-managed/1.0/single-cluster-helmfile" + destination: "/nvcf/overview/single-cluster-helmfile" + - source: "/nvcf/self-managed/1.0/state-metrics" + destination: "/nvcf/observability/1.0/state-metrics" + - source: "/nvcf/self-managed/1.0/topology-aware-scheduling" + destination: "/nvcf/compute-plane/1.0/topology-aware-scheduling" + - source: "/nvcf/self-managed/1.0/utils-container" + destination: "/nvcf/observability/1.0/utils-container" + - source: "/nvcf/self-managed/1.0/vault-open-bao" + destination: "/nvcf/observability/1.0/vault-open-bao" + - source: "/nvcf/self-managed/1.0/optional-enhancements" + destination: "/nvcf/self-managed/gateway-routing" + - source: "/nvcf/self-managed/1.0/runbooks" + destination: "/nvcf/self-managed/control-plane-operations" + - source: "/nvcf/self-managed/1.0/release-notes-overview" + destination: "/nvcf/overview/release-notes-overview" + - source: "/nvcf/self-managed/1.0/0-20-6-release-notes" + destination: "/nvcf/overview/1-0-release-notes" + - source: "/nvcf/self-managed/1.0/0-6-1-to-0-20-6-upgrade" + destination: "/nvcf/overview/0-6-1-to-1-0-upgrade" diff --git a/fern/products/compute-plane/1.0.yml b/fern/products/compute-plane/1.0.yml new file mode 100644 index 0000000000..1c7637c4f6 --- /dev/null +++ b/fern/products/compute-plane/1.0.yml @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +navigation: + - section: GPU Cluster Setup + skip-slug: true + contents: + - page: GPU Cluster Setup + path: ../../../docs/compute-plane-1.0/cluster-management/index.md + - page: Self-Managed Clusters + path: ../../../docs/compute-plane-1.0/cluster-management/self-managed.md + - page: Nsight Profiling + path: ../../../docs/compute-plane-1.0/cluster-management/nsight-profiling.md + - page: Fake GPU Operator + path: ../../../docs/compute-plane-1.0/fake-gpu-operator.md + + - section: Configuration + skip-slug: true + contents: + - page: Cluster Configuration + path: ../../../docs/compute-plane-1.0/cluster-management/configuration.md + - page: KAI Scheduler + path: ../../../docs/compute-plane-1.0/cluster-management/kai-scheduler.md + - page: Gang Scheduling + path: ../../../docs/compute-plane-1.0/cluster-management/gang-scheduling.md + - page: Topology-Aware Scheduling + path: ../../../docs/compute-plane-1.0/cluster-management/topology-aware-scheduling.md + + - section: Caching + skip-slug: true + contents: + - section: NVCF Caches + skip-slug: true + contents: + - page: Helm Model Caching + path: ../../../docs/compute-plane-1.0/cluster-management/model-cache.md + - page: Container Cache + path: ../../../docs/compute-plane-1.0/cluster-management/container-cache.md + - page: GXCache + path: ../../../docs/compute-plane-1.0/cluster-management/gxcache.md + - section: Physical Simulation Caches + skip-slug: true + contents: + - page: Simulation Caches + path: ../../../docs/compute-plane-1.0/caches.md + - section: DDCS + skip-slug: true + contents: + - link: DDCS + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/ + - link: DDCS Configuration + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/configure.html + - link: DDCS Deployment + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/deploy.html + - link: DDCS TLS + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/tls.html + - section: UCC + skip-slug: true + contents: + - link: UCC + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/ + - link: UCC Configuration + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/configure.html + - link: UCC Deployment + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/deploy.html + - link: UCC TLS + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/tls.html + - section: Cache Runbooks + skip-slug: true + contents: + - page: Caches Overview + path: ../../../docs/compute-plane-1.0/runbooks/caches.md + - link: DDCS Cache Misses + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/cache-misses.html + - link: DDCS Client Sharding + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/client-sharding.html + - link: DDCS Disk Exhaustion + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/disk-exhaustion.html + - link: DDCS Network Bottlenecks + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/network-bottlenecks.html + - link: DDCS RocksDB Corruption + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/rocksdb-corruption.html + - link: UCC Connection Saturation + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/connection-saturation.html + - link: UCC Data Disk Bandwidth + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/data-disk-bandwidth.html + - link: UCC Metadata Cache Undersizing + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/metadata-cache-undersizing.html + - link: UCC Network Bandwidth Saturation + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/network-bandwidth-saturation.html + - link: UCC Upstream S3 Connections + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/upstream-s3-connections.html + + - section: Operations + skip-slug: true + contents: + - page: Cluster Monitoring + path: ../../../docs/compute-plane-1.0/cluster-management/monitoring.md + + - section: Reference + skip-slug: true + contents: + - page: Cluster Reference + path: ../../../docs/compute-plane-1.0/cluster-management/reference.md diff --git a/fern/products/compute-plane/dev.yml b/fern/products/compute-plane/dev.yml new file mode 100644 index 0000000000..ee84624a81 --- /dev/null +++ b/fern/products/compute-plane/dev.yml @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +navigation: + - section: GPU Cluster Setup + skip-slug: true + contents: + - page: GPU Cluster Setup + path: ../../../docs/compute-plane/cluster-management/index.md + - page: Self-Managed Clusters + path: ../../../docs/compute-plane/cluster-management/self-managed.md + - page: Nsight Profiling + path: ../../../docs/compute-plane/cluster-management/nsight-profiling.md + - page: Fake GPU Operator + path: ../../../docs/compute-plane/fake-gpu-operator.md + + - section: Configuration + skip-slug: true + contents: + - page: Cluster Configuration + path: ../../../docs/compute-plane/cluster-management/configuration.md + - page: KAI Scheduler + path: ../../../docs/compute-plane/cluster-management/kai-scheduler.md + - page: Gang Scheduling + path: ../../../docs/compute-plane/cluster-management/gang-scheduling.md + - page: Topology-Aware Scheduling + path: ../../../docs/compute-plane/cluster-management/topology-aware-scheduling.md + + - section: Caching + skip-slug: true + contents: + - section: NVCF Caches + skip-slug: true + contents: + - page: Helm Model Caching + path: ../../../docs/compute-plane/cluster-management/model-cache.md + - page: Container Cache + path: ../../../docs/compute-plane/cluster-management/container-cache.md + - page: GXCache + path: ../../../docs/compute-plane/cluster-management/gxcache.md + - section: Physical Simulation Caches + skip-slug: true + contents: + - page: Simulation Caches + path: ../../../docs/compute-plane/caches.md + - section: DDCS + skip-slug: true + contents: + - link: DDCS + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/ + - link: DDCS Configuration + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/configure.html + - link: DDCS Deployment + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/deploy.html + - link: DDCS TLS + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/tls.html + - section: UCC + skip-slug: true + contents: + - link: UCC + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/ + - link: UCC Configuration + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/configure.html + - link: UCC Deployment + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/deploy.html + - link: UCC TLS + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/tls.html + - section: Cache Runbooks + skip-slug: true + contents: + - page: Caches Overview + path: ../../../docs/compute-plane/runbooks/caches.md + - link: DDCS Cache Misses + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/cache-misses.html + - link: DDCS Client Sharding + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/client-sharding.html + - link: DDCS Disk Exhaustion + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/disk-exhaustion.html + - link: DDCS Network Bottlenecks + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/network-bottlenecks.html + - link: DDCS RocksDB Corruption + href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/rocksdb-corruption.html + - link: UCC Connection Saturation + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/connection-saturation.html + - link: UCC Data Disk Bandwidth + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/data-disk-bandwidth.html + - link: UCC Metadata Cache Undersizing + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/metadata-cache-undersizing.html + - link: UCC Network Bandwidth Saturation + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/network-bandwidth-saturation.html + - link: UCC Upstream S3 Connections + href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/upstream-s3-connections.html + + - section: Operations + skip-slug: true + contents: + - page: Cluster Monitoring + path: ../../../docs/compute-plane/cluster-management/monitoring.md + + - section: Reference + skip-slug: true + contents: + - page: Cluster Reference + path: ../../../docs/compute-plane/cluster-management/reference.md diff --git a/fern/products/observability/1.0.yml b/fern/products/observability/1.0.yml new file mode 100644 index 0000000000..fa7f76ff93 --- /dev/null +++ b/fern/products/observability/1.0.yml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +navigation: + - page: Observability + path: ../../../docs/observability-1.0/observability.md + - page: Example Dashboards + path: ../../../docs/observability-1.0/example-dashboards.md + + - section: Metrics + skip-slug: true + contents: + - page: Metrics Overview + path: ../../../docs/observability-1.0/metrics/metrics-index.md + - page: Autoscaler + path: ../../../docs/observability-1.0/metrics/function-autoscaler/metrics.md + slug: autoscaler + - page: Cassandra + path: ../../../docs/observability-1.0/metrics/cassandra/metrics.md + - page: ESS + path: ../../../docs/observability-1.0/metrics/ess/metrics.md + - page: Init Container + path: ../../../docs/observability-1.0/metrics/init-container/metrics.md + - page: Invocation Service + path: ../../../docs/observability-1.0/metrics/invocation-service/metrics.md + - page: LLM API Gateway + path: ../../../docs/observability-1.0/metrics/llm-api-gateway/metrics.md + - page: LLM Function Invocation Metrics Report + path: ../../../docs/observability-1.0/metrics/llm-function-invocation-path.md + - page: LLM Request Router + path: ../../../docs/observability-1.0/metrics/llm-request-router/metrics.md + - page: NVCF API + path: ../../../docs/observability-1.0/metrics/nvcf-api/metrics.md + - page: SIS/Spot + path: ../../../docs/observability-1.0/metrics/sis-spot/metrics.md + - page: State Metrics + path: ../../../docs/observability-1.0/metrics/state-metrics/metrics.md + - page: Utils Container + path: ../../../docs/observability-1.0/metrics/utils-container/metrics.md + - page: Vault/OpenBao + path: ../../../docs/observability-1.0/metrics/vault-openbao/metrics.md diff --git a/fern/products/observability/dev.yml b/fern/products/observability/dev.yml new file mode 100644 index 0000000000..8ced640b31 --- /dev/null +++ b/fern/products/observability/dev.yml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +navigation: + - page: Observability + path: ../../../docs/observability/observability.md + - page: Example Dashboards + path: ../../../docs/observability/example-dashboards.md + + - section: Metrics + skip-slug: true + contents: + - page: Metrics Overview + path: ../../../docs/observability/metrics/metrics-index.md + - page: Autoscaler + path: ../../../docs/observability/metrics/function-autoscaler/metrics.md + slug: autoscaler + - page: Cassandra + path: ../../../docs/observability/metrics/cassandra/metrics.md + - page: ESS + path: ../../../docs/observability/metrics/ess/metrics.md + - page: Init Container + path: ../../../docs/observability/metrics/init-container/metrics.md + - page: Invocation Service + path: ../../../docs/observability/metrics/invocation-service/metrics.md + - page: LLM API Gateway + path: ../../../docs/observability/metrics/llm-api-gateway/metrics.md + - page: LLM Function Invocation Metrics Report + path: ../../../docs/observability/metrics/llm-function-invocation-path.md + - page: LLM Request Router + path: ../../../docs/observability/metrics/llm-request-router/metrics.md + - page: NVCF API + path: ../../../docs/observability/metrics/nvcf-api/metrics.md + - page: SIS/Spot + path: ../../../docs/observability/metrics/sis-spot/metrics.md + - page: State Metrics + path: ../../../docs/observability/metrics/state-metrics/metrics.md + - page: Utils Container + path: ../../../docs/observability/metrics/utils-container/metrics.md + - page: Vault/OpenBao + path: ../../../docs/observability/metrics/vault-openbao/metrics.md diff --git a/fern/products/overview.yml b/fern/products/overview.yml new file mode 100644 index 0000000000..221729df4c --- /dev/null +++ b/fern/products/overview.yml @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +navigation: + - page: Overview + path: ../../docs/overview/index.md + - link: Self-Managed Stack + href: /nvcf/self-managed/ + - link: Compute Plane Stack + href: /nvcf/compute-plane/ + - link: Observability Stack + href: /nvcf/observability/ + - page: Quickstart + path: ../../docs/overview/quickstart.md + + - section: Release Notes + skip-slug: true + contents: + - page: Release Notes Overview + path: ../../docs/overview/release-notes/index.md + - page: Compatibility Matrix + path: ../../docs/overview/compatibility-matrix.md + - page: "1.0 Release Notes" + path: ../../docs/overview/release-notes/1.0.md + - page: "0.6.1 to 1.0 Upgrade" + path: ../../docs/overview/release-notes/0.6.1-to-1.0-upgrade.md + - page: "(Legacy) 0.6.1 Release Notes" + path: ../../docs/overview/release-notes/0.6.1.md + - page: "(Legacy) 0.6.0 to 0.6.1 Upgrade" + path: ../../docs/overview/release-notes/0.6.0-to-0.6.1-upgrade.md + - page: "(Legacy) 0.6.0 Release Notes" + path: ../../docs/overview/release-notes/0.6.0.md + - page: "(Legacy) 0.5.0 to 0.6.0 Upgrade" + path: ../../docs/overview/release-notes/0.5.0-to-0.6.0-upgrade.md + + - section: Before You Deploy + skip-slug: true + contents: + - page: Infrastructure Sizing + path: ../../docs/overview/infrastructure-sizing.md + - page: Manifest + path: ../../docs/overview/manifest.md + - page: Image Mirroring + path: ../../docs/overview/image-mirroring.md + - page: Multi-Tenancy + path: ../../docs/overview/multi-tenancy.md + + - section: Using Cloud Functions + skip-slug: true + contents: + - page: API + path: ../../docs/overview/api.md + - page: Function Creation + path: ../../docs/overview/function-creation.md + - page: LLM Gateway + path: ../../docs/overview/llm-gateway.md + - page: Generic HTTP Function Invocation + path: ../../docs/overview/generic-http-function-invocation.md + - page: gRPC Function Invocation + path: ../../docs/overview/grpc-function-invocation.md + - page: Container Functions + path: ../../docs/overview/container-functions.md + - page: Helm Functions + path: ../../docs/overview/helm-functions.md + - page: Streaming Functions + path: ../../docs/overview/streaming-functions.md + - page: Configure Autoscaling + path: ../../docs/overview/configure-autoscaling.md + - page: CLI + path: ../../docs/overview/cli.md + - page: Task Creation + path: ../../docs/overview/task-creation.md + - page: Container-Based Task Creation + path: ../../docs/overview/container-tasks.md + - page: Helm-Based Task Creation + path: ../../docs/overview/helm-tasks.md + + - section: Load Testing + skip-slug: true + contents: + - page: gRPC Load Testing + path: ../../docs/overview/grpc-load-testing.md + - page: gRPC Load Test SLI Guide + path: ../../docs/overview/grpc-load-test-sli-guide.md + - page: HTTP Load Testing + path: ../../docs/overview/http-load-testing.md + - page: HTTP Load Test SLI Guide + path: ../../docs/overview/http-load-test-sli-guide.md + - page: HTTP Soak Testing + path: ../../docs/overview/http-soak-testing.md + + - section: Development + skip-slug: true + contents: + - page: Architecture Overview + path: ../../docs/dev/architecture.md + - section: Local Development + skip-slug: true + contents: + - page: Local Development + path: ../../docs/overview/local-development.md + - page: Single-cluster (CLI) + path: ../../docs/overview/local-development/single-cluster-cli.md + - page: Single-cluster (Helmfile) + path: ../../docs/overview/local-development/single-cluster-helmfile.md + - page: Multi-cluster (CLI) + path: ../../docs/overview/local-development/multi-cluster-cli.md + - page: Multi-cluster (Helmfile) + path: ../../docs/overview/local-development/multi-cluster-helmfile.md + + - section: Managed (Legacy) + skip-slug: true + contents: + - section: Cluster Management + skip-slug: true + contents: + - page: NGC-Managed Clusters + path: ../../docs/ngc-managed/cluster-management/ngc-managed.md + - page: Helm-Managed Clusters + path: ../../docs/ngc-managed/cluster-management/helm-managed.md + - page: GitOps Cluster Registration + path: ../../docs/ngc-managed/cluster-management/gitops-registration.md + - page: Configuration + path: ../../docs/ngc-managed/cluster-management/configuration.md + - page: Monitoring + path: ../../docs/ngc-managed/cluster-management/monitoring.md + - page: Reference + path: ../../docs/ngc-managed/cluster-management/reference.md + - page: Container Cache + path: ../../docs/ngc-managed/cluster-management/container-cache.md + - page: GXCache + path: ../../docs/ngc-managed/cluster-management/gxcache.md + - page: KAI Scheduler + path: ../../docs/ngc-managed/cluster-management/kai-scheduler.md + - page: Function Lifecycle + path: ../../docs/ngc-managed/function-lifecycle.md + - page: Service Keys + path: ../../docs/ngc-managed/service-keys.md + - page: Observability + slug: managed-observability + path: ../../docs/ngc-managed/observability.md diff --git a/fern/products/self-managed/1.0.yml b/fern/products/self-managed/1.0.yml new file mode 100644 index 0000000000..21a7a8e264 --- /dev/null +++ b/fern/products/self-managed/1.0.yml @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +navigation: + - section: Deployment + skip-slug: true + contents: + - page: Installation Overview + path: ../../../docs/self-managed-1.0/installation.md + - page: Helmfile Installation + path: ../../../docs/self-managed-1.0/helmfile-installation.md + - page: CSP End-to-End Example + path: ../../../docs/self-managed-1.0/csp-end-to-end-example-installation.md + + - section: Configuration + skip-slug: true + contents: + - page: NVCF UI + path: ../../../docs/self-managed-1.0/nvcf-ui.md + - page: Pod Disruption Budgets + path: ../../../docs/self-managed-1.0/pod-disruption-budgets.md + - page: gRPC Invocation Enablement + path: ../../../docs/self-managed-1.0/grpc-invocation-enablement.md + - page: LLM Function Enablement + path: ../../../docs/self-managed-1.0/llm-function-enablement.md + - page: LLM Request Router Load Balancing + path: ../../../docs/self-managed-1.0/llm-request-router-load-balancing.md + - page: Gateway Routing + path: ../../../docs/self-managed-1.0/gateway-routing.md + - page: Third-Party Registries + path: ../../../docs/self-managed-1.0/third-party-registries.md + - page: Registry Allowlist + path: ../../../docs/self-managed-1.0/registry-allowlist.md + - section: Low Latency Streaming + skip-slug: true + contents: + - page: LLS Installation + path: ../../../docs/self-managed-1.0/lls-installation.md + + - section: Function Autoscaling + skip-slug: true + contents: + - page: Function Autoscaling Overview + path: ../../../docs/self-managed-1.0/autoscaling/index.md + - page: Architecture + path: ../../../docs/self-managed-1.0/autoscaling/architecture.md + - page: Operations + path: ../../../docs/self-managed-1.0/autoscaling/operations.md + + - section: Operations + skip-slug: true + contents: + - page: Control Plane Operations + path: ../../../docs/self-managed-1.0/control-plane-operations.md + - page: Troubleshooting + path: ../../../docs/self-managed-1.0/troubleshooting.md + - page: Key Rotation + path: ../../../docs/self-managed-1.0/runbooks/control-plane-key-rotation-mek.md + - page: Transport TLS Rotation + path: ../../../docs/self-managed-1.0/runbooks/transport-tls-rotation.md + - page: Quota Management + path: ../../../docs/self-managed-1.0/runbooks/update-nvcf-account-quota-limits.md diff --git a/fern/products/self-managed/dev.yml b/fern/products/self-managed/dev.yml new file mode 100644 index 0000000000..f8eb7257fb --- /dev/null +++ b/fern/products/self-managed/dev.yml @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +navigation: + - section: Deployment + skip-slug: true + contents: + - page: Installation Overview + path: ../../../docs/self-managed/installation.md + - page: Helmfile Installation + path: ../../../docs/self-managed/helmfile-installation.md + - page: CSP End-to-End Example + path: ../../../docs/self-managed/csp-end-to-end-example-installation.md + + - section: Configuration + skip-slug: true + contents: + - page: NVCF UI + path: ../../../docs/self-managed/nvcf-ui.md + - page: Pod Disruption Budgets + path: ../../../docs/self-managed/pod-disruption-budgets.md + - page: gRPC Invocation Enablement + path: ../../../docs/self-managed/grpc-invocation-enablement.md + - page: LLM Function Enablement + path: ../../../docs/self-managed/llm-function-enablement.md + - page: LLM Request Router Load Balancing + path: ../../../docs/self-managed/llm-request-router-load-balancing.md + - page: Gateway Routing + path: ../../../docs/self-managed/gateway-routing.md + - page: Third-Party Registries + path: ../../../docs/self-managed/third-party-registries.md + - page: Registry Allowlist + path: ../../../docs/self-managed/registry-allowlist.md + - section: Low Latency Streaming + skip-slug: true + contents: + - page: LLS Installation + path: ../../../docs/self-managed/lls-installation.md + + - section: Function Autoscaling + skip-slug: true + contents: + - page: Function Autoscaling Overview + path: ../../../docs/self-managed/autoscaling/index.md + - page: Architecture + path: ../../../docs/self-managed/autoscaling/architecture.md + - page: Operations + path: ../../../docs/self-managed/autoscaling/operations.md + + - section: Operations + skip-slug: true + contents: + - page: Control Plane Operations + path: ../../../docs/self-managed/control-plane-operations.md + - page: Troubleshooting + path: ../../../docs/self-managed/troubleshooting.md + - page: Key Rotation + path: ../../../docs/self-managed/runbooks/control-plane-key-rotation-mek.md + - page: Transport TLS Rotation + path: ../../../docs/self-managed/runbooks/transport-tls-rotation.md + - page: Quota Management + path: ../../../docs/self-managed/runbooks/update-nvcf-account-quota-limits.md diff --git a/fern/versions/v0.5.yml b/fern/products/self-managed/v0.5.yml similarity index 61% rename from fern/versions/v0.5.yml rename to fern/products/self-managed/v0.5.yml index 9a5935d06e..e5855e8455 100644 --- a/fern/versions/v0.5.yml +++ b/fern/products/self-managed/v0.5.yml @@ -3,79 +3,79 @@ navigation: - page: Overview - path: ../../docs/v0.5/index.md + path: ../../../docs/v0.5/index.md - section: Before You Deploy skip-slug: true contents: - page: Infrastructure Sizing - path: ../../docs/v0.5/infrastructure-sizing.md + path: ../../../docs/v0.5/infrastructure-sizing.md - page: Manifest - path: ../../docs/v0.5/manifest.md + path: ../../../docs/v0.5/manifest.md - section: Deployment skip-slug: true contents: - page: Installation Overview - path: ../../docs/v0.5/installation.md + path: ../../../docs/v0.5/installation.md - page: Image Mirroring - path: ../../docs/v0.5/image-mirroring.md + path: ../../../docs/v0.5/image-mirroring.md - page: Terraform Installation - path: ../../docs/v0.5/terraform-installation.md + path: ../../../docs/v0.5/terraform-installation.md - page: Helmfile Installation - path: ../../docs/v0.5/helmfile-installation.md + path: ../../../docs/v0.5/helmfile-installation.md - section: Standalone Deployment skip-slug: true contents: - page: Overview - path: ../../docs/v0.5/standalone-deployment.md + path: ../../../docs/v0.5/standalone-deployment.md - page: Prerequisites - path: ../../docs/v0.5/standalone-prerequisites.md + path: ../../../docs/v0.5/standalone-prerequisites.md - page: Infrastructure - path: ../../docs/v0.5/standalone-infrastructure.md + path: ../../../docs/v0.5/standalone-infrastructure.md - page: Core Services - path: ../../docs/v0.5/standalone-core-services.md + path: ../../../docs/v0.5/standalone-core-services.md - page: Gateway - path: ../../docs/v0.5/standalone-gateway.md + path: ../../../docs/v0.5/standalone-gateway.md - section: GPU Cluster Setup skip-slug: true contents: - page: GPU Cluster Setup - path: ../../docs/v0.5/cluster-management/index.md + path: ../../../docs/v0.5/cluster-management/index.md - page: Self-Managed Clusters - path: ../../docs/v0.5/cluster-management/self-managed.md + path: ../../../docs/v0.5/cluster-management/self-managed.md - section: Configuration skip-slug: true contents: - page: Optional Enhancements - path: ../../docs/v0.5/optional-enhancements.md + path: ../../../docs/v0.5/optional-enhancements.md - page: Gateway Routing - path: ../../docs/v0.5/gateway-routing.md + path: ../../../docs/v0.5/gateway-routing.md - page: Third-Party Registries - path: ../../docs/v0.5/third-party-registries.md + path: ../../../docs/v0.5/third-party-registries.md - page: Cluster Configuration - path: ../../docs/v0.5/cluster-management/configuration.md + path: ../../../docs/v0.5/cluster-management/configuration.md - page: KAI Scheduler - path: ../../docs/v0.5/cluster-management/kai-scheduler.md + path: ../../../docs/v0.5/cluster-management/kai-scheduler.md - section: Low Latency Streaming skip-slug: true contents: - page: LLS Installation - path: ../../docs/v0.5/lls-installation.md + path: ../../../docs/v0.5/lls-installation.md - section: NVCF Caches skip-slug: true contents: - page: Container Cache - path: ../../docs/v0.5/cluster-management/container-cache.md + path: ../../../docs/v0.5/cluster-management/container-cache.md - page: GXCache - path: ../../docs/v0.5/cluster-management/gxcache.md + path: ../../../docs/v0.5/cluster-management/gxcache.md - section: Physical Simulation Caches skip-slug: true contents: - page: Simulation Caches - path: ../../docs/v0.5/caches.md + path: ../../../docs/v0.5/caches.md - section: DDCS skip-slug: true contents: @@ -99,75 +99,59 @@ navigation: - link: UCC TLS href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/tls.html - - section: Using Cloud Functions - skip-slug: true - contents: - - page: API - path: ../../docs/v0.5/api.md - - page: Function Creation - path: ../../docs/v0.5/function-creation.md - - page: Container Functions - path: ../../docs/v0.5/container-functions.md - - page: Helm Functions - path: ../../docs/v0.5/helm-functions.md - - page: Streaming Functions - path: ../../docs/v0.5/streaming-functions.md - - page: CLI - path: ../../docs/v0.5/cli.md - - section: Observability skip-slug: true contents: - page: Observability - path: ../../docs/v0.5/observability.md + path: ../../../docs/v0.5/observability.md - page: Example Dashboards - path: ../../docs/v0.5/example-dashboards.md + path: ../../../docs/v0.5/example-dashboards.md - section: Metrics skip-slug: true contents: - page: Overview - path: ../../docs/v0.5/metrics/metrics-index.md + path: ../../../docs/v0.5/metrics/metrics-index.md - page: Cassandra - path: ../../docs/v0.5/metrics/cassandra/metrics.md + path: ../../../docs/v0.5/metrics/cassandra/metrics.md - page: ESS - path: ../../docs/v0.5/metrics/ess/metrics.md + path: ../../../docs/v0.5/metrics/ess/metrics.md - page: Init Container - path: ../../docs/v0.5/metrics/init-container/metrics.md + path: ../../../docs/v0.5/metrics/init-container/metrics.md - page: Invocation Service - path: ../../docs/v0.5/metrics/invocation-service/metrics.md + path: ../../../docs/v0.5/metrics/invocation-service/metrics.md - page: NVCF API - path: ../../docs/v0.5/metrics/nvcf-api/metrics.md + path: ../../../docs/v0.5/metrics/nvcf-api/metrics.md - page: SIS/Spot - path: ../../docs/v0.5/metrics/sis-spot/metrics.md + path: ../../../docs/v0.5/metrics/sis-spot/metrics.md - page: State Metrics - path: ../../docs/v0.5/metrics/state-metrics/metrics.md + path: ../../../docs/v0.5/metrics/state-metrics/metrics.md - page: Utils Container - path: ../../docs/v0.5/metrics/utils-container/metrics.md + path: ../../../docs/v0.5/metrics/utils-container/metrics.md - page: Vault/OpenBao - path: ../../docs/v0.5/metrics/vault-openbao/metrics.md + path: ../../../docs/v0.5/metrics/vault-openbao/metrics.md - section: Operations skip-slug: true contents: - page: Control Plane Operations - path: ../../docs/v0.5/control-plane-operations.md + path: ../../../docs/v0.5/control-plane-operations.md - page: Cluster Monitoring - path: ../../docs/v0.5/cluster-management/monitoring.md + path: ../../../docs/v0.5/cluster-management/monitoring.md - page: Troubleshooting - path: ../../docs/v0.5/troubleshooting.md + path: ../../../docs/v0.5/troubleshooting.md - section: Runbooks skip-slug: true contents: - page: Runbooks - path: ../../docs/v0.5/runbooks/index.md + path: ../../../docs/v0.5/runbooks/index.md - page: Key Rotation - path: ../../docs/v0.5/runbooks/control-plane-key-rotation-mek.md + path: ../../../docs/v0.5/runbooks/control-plane-key-rotation-mek.md - section: Cache Runbooks skip-slug: true contents: - page: Caches Overview - path: ../../docs/v0.5/runbooks/caches.md + path: ../../../docs/v0.5/runbooks/caches.md - link: DDCS Cache Misses href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/cache-misses.html - link: DDCS Client Sharding @@ -193,16 +177,12 @@ navigation: skip-slug: true contents: - page: Cluster Reference - path: ../../docs/v0.5/cluster-management/reference.md - - page: gRPC Load Testing - path: ../../docs/v0.5/grpc-load-testing.md - - page: HTTP Soak Testing - path: ../../docs/v0.5/http-soak-testing.md + path: ../../../docs/v0.5/cluster-management/reference.md - section: Development skip-slug: true contents: - page: Local Development - path: ../../docs/v0.5/local-development.md + path: ../../../docs/v0.5/local-development.md - page: Fake GPU Operator - path: ../../docs/v0.5/fake-gpu-operator.md + path: ../../../docs/v0.5/fake-gpu-operator.md diff --git a/fern/versions/v0.6.0.yml b/fern/products/self-managed/v0.6.0.yml similarity index 52% rename from fern/versions/v0.6.0.yml rename to fern/products/self-managed/v0.6.0.yml index cbc4a417fa..a8a0600a39 100644 --- a/fern/versions/v0.6.0.yml +++ b/fern/products/self-managed/v0.6.0.yml @@ -3,84 +3,84 @@ navigation: - page: Overview - path: ../../docs/v0.6.0/index.md + path: ../../../docs/v0.6.0/index.md - page: Quickstart - path: ../../docs/v0.6.0/quickstart.md + path: ../../../docs/v0.6.0/quickstart.md - section: Release Notes skip-slug: true contents: - page: Release Notes Overview - path: ../../docs/v0.6.0/release-notes/index.md + path: ../../../docs/v0.6.0/release-notes/index.md - page: "0.6.0 Release Notes" - path: ../../docs/v0.6.0/release-notes/0.6.0.md + path: ../../../docs/v0.6.0/release-notes/0.6.0.md - page: "0.5.0 to 0.6.0 Upgrade" - path: ../../docs/v0.6.0/release-notes/0.5.0-to-0.6.0-upgrade.md + path: ../../../docs/v0.6.0/release-notes/0.5.0-to-0.6.0-upgrade.md - section: Before You Deploy skip-slug: true contents: - page: Infrastructure Sizing - path: ../../docs/v0.6.0/infrastructure-sizing.md + path: ../../../docs/v0.6.0/infrastructure-sizing.md - page: Manifest - path: ../../docs/v0.6.0/manifest.md + path: ../../../docs/v0.6.0/manifest.md - section: Deployment skip-slug: true contents: - page: Installation Overview - path: ../../docs/v0.6.0/installation.md + path: ../../../docs/v0.6.0/installation.md - page: Image Mirroring - path: ../../docs/v0.6.0/image-mirroring.md + path: ../../../docs/v0.6.0/image-mirroring.md - page: Helmfile Installation - path: ../../docs/v0.6.0/helmfile-installation.md + path: ../../../docs/v0.6.0/helmfile-installation.md - page: gRPC Invocation Enablement - path: ../../docs/v0.6.0/grpc-invocation-enablement.md + path: ../../../docs/v0.6.0/grpc-invocation-enablement.md - page: CSP End-to-End Example - path: ../../docs/v0.6.0/csp-end-to-end-example-installation.md + path: ../../../docs/v0.6.0/csp-end-to-end-example-installation.md - section: GPU Cluster Setup skip-slug: true contents: - page: GPU Cluster Setup - path: ../../docs/v0.6.0/cluster-management/index.md + path: ../../../docs/v0.6.0/cluster-management/index.md - page: Self-Managed Clusters - path: ../../docs/v0.6.0/cluster-management/self-managed.md + path: ../../../docs/v0.6.0/cluster-management/self-managed.md - section: Configuration skip-slug: true contents: - page: Optional Enhancements - path: ../../docs/v0.6.0/optional-enhancements.md + path: ../../../docs/v0.6.0/optional-enhancements.md - page: LLM Function Enablement - path: ../../docs/v0.6.0/llm-function-enablement.md + path: ../../../docs/v0.6.0/llm-function-enablement.md - page: Gateway Routing - path: ../../docs/v0.6.0/gateway-routing.md + path: ../../../docs/v0.6.0/gateway-routing.md - page: Third-Party Registries - path: ../../docs/v0.6.0/third-party-registries.md + path: ../../../docs/v0.6.0/third-party-registries.md - page: Registry Allowlist - path: ../../docs/v0.6.0/registry-allowlist.md + path: ../../../docs/v0.6.0/registry-allowlist.md - page: Cluster Configuration - path: ../../docs/v0.6.0/cluster-management/configuration.md + path: ../../../docs/v0.6.0/cluster-management/configuration.md - page: KAI Scheduler - path: ../../docs/v0.6.0/cluster-management/kai-scheduler.md + path: ../../../docs/v0.6.0/cluster-management/kai-scheduler.md - section: Low Latency Streaming skip-slug: true contents: - page: LLS Installation - path: ../../docs/v0.6.0/lls-installation.md + path: ../../../docs/v0.6.0/lls-installation.md - section: NVCF Caches skip-slug: true contents: - page: Container Cache - path: ../../docs/v0.6.0/cluster-management/container-cache.md + path: ../../../docs/v0.6.0/cluster-management/container-cache.md - page: GXCache - path: ../../docs/v0.6.0/cluster-management/gxcache.md + path: ../../../docs/v0.6.0/cluster-management/gxcache.md - section: Physical Simulation Caches skip-slug: true contents: - page: Simulation Caches - path: ../../docs/v0.6.0/caches.md + path: ../../../docs/v0.6.0/caches.md - section: DDCS skip-slug: true contents: @@ -104,108 +104,78 @@ navigation: - link: UCC TLS href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/tls.html - - section: Using Cloud Functions - skip-slug: true - contents: - - page: API - path: ../../docs/v0.6.0/api.md - - page: Function Creation - path: ../../docs/v0.6.0/function-creation.md - - page: LLM Gateway - path: ../../docs/v0.6.0/llm-gateway.md - - page: Generic HTTP Function Invocation - path: ../../docs/v0.6.0/generic-http-function-invocation.md - - page: gRPC Function Invocation - path: ../../docs/v0.6.0/grpc-function-invocation.md - - page: Container Functions - path: ../../docs/v0.6.0/container-functions.md - - page: Helm Functions - path: ../../docs/v0.6.0/helm-functions.md - - page: Streaming Functions - path: ../../docs/v0.6.0/streaming-functions.md - - page: Configure Autoscaling - path: ../../docs/v0.6.0/configure-autoscaling.md - - page: CLI - path: ../../docs/v0.6.0/cli.md - - page: Task Creation - path: ../../docs/v0.6.0/task-creation.md - - page: Container-Based Task Creation - path: ../../docs/v0.6.0/container-tasks.md - - page: Helm-Based Task Creation - path: ../../docs/v0.6.0/helm-tasks.md - - section: Function Autoscaling skip-slug: true contents: - page: Function Autoscaling Overview - path: ../../docs/v0.6.0/autoscaling/index.md + path: ../../../docs/v0.6.0/autoscaling/index.md - page: Architecture - path: ../../docs/v0.6.0/autoscaling/architecture.md + path: ../../../docs/v0.6.0/autoscaling/architecture.md - page: Operations - path: ../../docs/v0.6.0/autoscaling/operations.md + path: ../../../docs/v0.6.0/autoscaling/operations.md - page: Observability - path: ../../docs/v0.6.0/autoscaling/observability.md + path: ../../../docs/v0.6.0/autoscaling/observability.md slug: autoscaler-observability - section: Observability skip-slug: true contents: - page: Observability - path: ../../docs/v0.6.0/observability.md + path: ../../../docs/v0.6.0/observability.md - page: Example Dashboards - path: ../../docs/v0.6.0/example-dashboards.md + path: ../../../docs/v0.6.0/example-dashboards.md - section: Metrics skip-slug: true contents: - page: Metrics Overview - path: ../../docs/v0.6.0/metrics/metrics-index.md + path: ../../../docs/v0.6.0/metrics/metrics-index.md - page: Cassandra - path: ../../docs/v0.6.0/metrics/cassandra/metrics.md + path: ../../../docs/v0.6.0/metrics/cassandra/metrics.md - page: ESS - path: ../../docs/v0.6.0/metrics/ess/metrics.md + path: ../../../docs/v0.6.0/metrics/ess/metrics.md - page: Init Container - path: ../../docs/v0.6.0/metrics/init-container/metrics.md + path: ../../../docs/v0.6.0/metrics/init-container/metrics.md - page: Invocation Service - path: ../../docs/v0.6.0/metrics/invocation-service/metrics.md + path: ../../../docs/v0.6.0/metrics/invocation-service/metrics.md - page: LLM API Gateway - path: ../../docs/v0.6.0/metrics/llm-api-gateway/metrics.md + path: ../../../docs/v0.6.0/metrics/llm-api-gateway/metrics.md - page: LLM Function Invocation Metrics Report - path: ../../docs/v0.6.0/metrics/llm-function-invocation-path.md + path: ../../../docs/v0.6.0/metrics/llm-function-invocation-path.md - page: LLM Request Router - path: ../../docs/v0.6.0/metrics/llm-request-router/metrics.md + path: ../../../docs/v0.6.0/metrics/llm-request-router/metrics.md - page: NVCF API - path: ../../docs/v0.6.0/metrics/nvcf-api/metrics.md + path: ../../../docs/v0.6.0/metrics/nvcf-api/metrics.md - page: SIS/Spot - path: ../../docs/v0.6.0/metrics/sis-spot/metrics.md + path: ../../../docs/v0.6.0/metrics/sis-spot/metrics.md - page: State Metrics - path: ../../docs/v0.6.0/metrics/state-metrics/metrics.md + path: ../../../docs/v0.6.0/metrics/state-metrics/metrics.md - page: Utils Container - path: ../../docs/v0.6.0/metrics/utils-container/metrics.md + path: ../../../docs/v0.6.0/metrics/utils-container/metrics.md - page: Vault/OpenBao - path: ../../docs/v0.6.0/metrics/vault-openbao/metrics.md + path: ../../../docs/v0.6.0/metrics/vault-openbao/metrics.md - section: Operations skip-slug: true contents: - page: Control Plane Operations - path: ../../docs/v0.6.0/control-plane-operations.md + path: ../../../docs/v0.6.0/control-plane-operations.md - page: Cluster Monitoring - path: ../../docs/v0.6.0/cluster-management/monitoring.md + path: ../../../docs/v0.6.0/cluster-management/monitoring.md - page: Troubleshooting - path: ../../docs/v0.6.0/troubleshooting.md + path: ../../../docs/v0.6.0/troubleshooting.md - section: Runbooks skip-slug: true contents: - page: Runbooks - path: ../../docs/v0.6.0/runbooks/index.md + path: ../../../docs/v0.6.0/runbooks/index.md - page: Key Rotation - path: ../../docs/v0.6.0/runbooks/control-plane-key-rotation-mek.md + path: ../../../docs/v0.6.0/runbooks/control-plane-key-rotation-mek.md - section: Cache Runbooks skip-slug: true contents: - page: Caches Overview - path: ../../docs/v0.6.0/runbooks/caches.md + path: ../../../docs/v0.6.0/runbooks/caches.md - link: DDCS Cache Misses href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/cache-misses.html - link: DDCS Client Sharding @@ -231,40 +201,30 @@ navigation: skip-slug: true contents: - page: Cluster Reference - path: ../../docs/v0.6.0/cluster-management/reference.md - - page: gRPC Load Testing - path: ../../docs/v0.6.0/grpc-load-testing.md - - page: gRPC Load Test SLI Guide - path: ../../docs/v0.6.0/grpc-load-test-sli-guide.md - - page: HTTP Load Testing - path: ../../docs/v0.6.0/http-load-testing.md - - page: HTTP Load Test SLI Guide - path: ../../docs/v0.6.0/http-load-test-sli-guide.md - - page: HTTP Soak Testing - path: ../../docs/v0.6.0/http-soak-testing.md + path: ../../../docs/v0.6.0/cluster-management/reference.md - section: Development skip-slug: true contents: - page: Architecture Overview - path: ../../docs/dev/architecture.md + path: ../../../docs/dev/architecture.md - section: Local Development skip-slug: true contents: - page: Local Development - path: ../../docs/v0.6.0/local-development.md + path: ../../../docs/v0.6.0/local-development.md - page: Single-cluster (CLI) - path: ../../docs/v0.6.0/local-development/single-cluster-cli.md + path: ../../../docs/v0.6.0/local-development/single-cluster-cli.md - page: Single-cluster (Helmfile) - path: ../../docs/v0.6.0/local-development/single-cluster-helmfile.md + path: ../../../docs/v0.6.0/local-development/single-cluster-helmfile.md - page: Multi-cluster (CLI) - path: ../../docs/v0.6.0/local-development/multi-cluster-cli.md + path: ../../../docs/v0.6.0/local-development/multi-cluster-cli.md - page: Multi-cluster (Helmfile) - path: ../../docs/v0.6.0/local-development/multi-cluster-helmfile.md + path: ../../../docs/v0.6.0/local-development/multi-cluster-helmfile.md - page: Fake GPU Operator - path: ../../docs/v0.6.0/fake-gpu-operator.md + path: ../../../docs/v0.6.0/fake-gpu-operator.md - page: Release Process - path: ../../docs/dev/release-process.md + path: ../../../docs/dev/release-process.md - section: Managed (Legacy) skip-slug: true @@ -273,25 +233,25 @@ navigation: skip-slug: true contents: - page: NGC-Managed Clusters - path: ../../docs/ngc-managed/cluster-management/ngc-managed.md + path: ../../../docs/ngc-managed/cluster-management/ngc-managed.md - page: Helm-Managed Clusters - path: ../../docs/ngc-managed/cluster-management/helm-managed.md + path: ../../../docs/ngc-managed/cluster-management/helm-managed.md - page: Configuration - path: ../../docs/ngc-managed/cluster-management/configuration.md + path: ../../../docs/ngc-managed/cluster-management/configuration.md - page: Monitoring - path: ../../docs/ngc-managed/cluster-management/monitoring.md + path: ../../../docs/ngc-managed/cluster-management/monitoring.md - page: Reference - path: ../../docs/ngc-managed/cluster-management/reference.md + path: ../../../docs/ngc-managed/cluster-management/reference.md - page: Container Cache - path: ../../docs/ngc-managed/cluster-management/container-cache.md + path: ../../../docs/ngc-managed/cluster-management/container-cache.md - page: GXCache - path: ../../docs/ngc-managed/cluster-management/gxcache.md + path: ../../../docs/ngc-managed/cluster-management/gxcache.md - page: KAI Scheduler - path: ../../docs/ngc-managed/cluster-management/kai-scheduler.md + path: ../../../docs/ngc-managed/cluster-management/kai-scheduler.md - page: Function Lifecycle - path: ../../docs/ngc-managed/function-lifecycle.md + path: ../../../docs/ngc-managed/function-lifecycle.md - page: Service Keys - path: ../../docs/ngc-managed/service-keys.md + path: ../../../docs/ngc-managed/service-keys.md - page: Observability slug: managed-observability - path: ../../docs/ngc-managed/observability.md + path: ../../../docs/ngc-managed/observability.md diff --git a/fern/versions/v0.6.1.yml b/fern/products/self-managed/v0.6.1.yml similarity index 51% rename from fern/versions/v0.6.1.yml rename to fern/products/self-managed/v0.6.1.yml index 55910b3b37..5b3ea0b187 100644 --- a/fern/versions/v0.6.1.yml +++ b/fern/products/self-managed/v0.6.1.yml @@ -3,88 +3,88 @@ navigation: - page: Overview - path: ../../docs/v0.6.1/index.md + path: ../../../docs/v0.6.1/index.md - page: Quickstart - path: ../../docs/v0.6.1/quickstart.md + path: ../../../docs/v0.6.1/quickstart.md - section: Release Notes skip-slug: true contents: - page: Release Notes Overview - path: ../../docs/v0.6.1/release-notes/index.md + path: ../../../docs/v0.6.1/release-notes/index.md - page: "0.6.1 Release Notes" - path: ../../docs/v0.6.1/release-notes/0.6.1.md + path: ../../../docs/v0.6.1/release-notes/0.6.1.md - page: "0.6.0 to 0.6.1 Upgrade" - path: ../../docs/v0.6.1/release-notes/0.6.0-to-0.6.1-upgrade.md + path: ../../../docs/v0.6.1/release-notes/0.6.0-to-0.6.1-upgrade.md - page: "0.6.0 Release Notes" - path: ../../docs/v0.6.1/release-notes/0.6.0.md + path: ../../../docs/v0.6.1/release-notes/0.6.0.md - page: "0.5.0 to 0.6.0 Upgrade" - path: ../../docs/v0.6.1/release-notes/0.5.0-to-0.6.0-upgrade.md + path: ../../../docs/v0.6.1/release-notes/0.5.0-to-0.6.0-upgrade.md - section: Before You Deploy skip-slug: true contents: - page: Infrastructure Sizing - path: ../../docs/v0.6.1/infrastructure-sizing.md + path: ../../../docs/v0.6.1/infrastructure-sizing.md - page: Manifest - path: ../../docs/v0.6.1/manifest.md + path: ../../../docs/v0.6.1/manifest.md - section: Deployment skip-slug: true contents: - page: Installation Overview - path: ../../docs/v0.6.1/installation.md + path: ../../../docs/v0.6.1/installation.md - page: Image Mirroring - path: ../../docs/v0.6.1/image-mirroring.md + path: ../../../docs/v0.6.1/image-mirroring.md - page: Helmfile Installation - path: ../../docs/v0.6.1/helmfile-installation.md + path: ../../../docs/v0.6.1/helmfile-installation.md - page: gRPC Invocation Enablement - path: ../../docs/v0.6.1/grpc-invocation-enablement.md + path: ../../../docs/v0.6.1/grpc-invocation-enablement.md - page: CSP End-to-End Example - path: ../../docs/v0.6.1/csp-end-to-end-example-installation.md + path: ../../../docs/v0.6.1/csp-end-to-end-example-installation.md - section: GPU Cluster Setup skip-slug: true contents: - page: GPU Cluster Setup - path: ../../docs/v0.6.1/cluster-management/index.md + path: ../../../docs/v0.6.1/cluster-management/index.md - page: Self-Managed Clusters - path: ../../docs/v0.6.1/cluster-management/self-managed.md + path: ../../../docs/v0.6.1/cluster-management/self-managed.md - section: Configuration skip-slug: true contents: - page: Optional Enhancements - path: ../../docs/v0.6.1/optional-enhancements.md + path: ../../../docs/v0.6.1/optional-enhancements.md - page: LLM Function Enablement - path: ../../docs/v0.6.1/llm-function-enablement.md + path: ../../../docs/v0.6.1/llm-function-enablement.md - page: Gateway Routing - path: ../../docs/v0.6.1/gateway-routing.md + path: ../../../docs/v0.6.1/gateway-routing.md - page: Third-Party Registries - path: ../../docs/v0.6.1/third-party-registries.md + path: ../../../docs/v0.6.1/third-party-registries.md - page: Registry Allowlist - path: ../../docs/v0.6.1/registry-allowlist.md + path: ../../../docs/v0.6.1/registry-allowlist.md - page: Cluster Configuration - path: ../../docs/v0.6.1/cluster-management/configuration.md + path: ../../../docs/v0.6.1/cluster-management/configuration.md - page: KAI Scheduler - path: ../../docs/v0.6.1/cluster-management/kai-scheduler.md + path: ../../../docs/v0.6.1/cluster-management/kai-scheduler.md - section: Low Latency Streaming skip-slug: true contents: - page: LLS Installation - path: ../../docs/v0.6.1/lls-installation.md + path: ../../../docs/v0.6.1/lls-installation.md - section: NVCF Caches skip-slug: true contents: - page: Container Cache - path: ../../docs/v0.6.1/cluster-management/container-cache.md + path: ../../../docs/v0.6.1/cluster-management/container-cache.md - page: GXCache - path: ../../docs/v0.6.1/cluster-management/gxcache.md + path: ../../../docs/v0.6.1/cluster-management/gxcache.md - section: Physical Simulation Caches skip-slug: true contents: - page: Simulation Caches - path: ../../docs/v0.6.1/caches.md + path: ../../../docs/v0.6.1/caches.md - section: DDCS skip-slug: true contents: @@ -108,108 +108,78 @@ navigation: - link: UCC TLS href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/tls.html - - section: Using Cloud Functions - skip-slug: true - contents: - - page: API - path: ../../docs/v0.6.1/api.md - - page: Function Creation - path: ../../docs/v0.6.1/function-creation.md - - page: LLM Gateway - path: ../../docs/v0.6.1/llm-gateway.md - - page: Generic HTTP Function Invocation - path: ../../docs/v0.6.1/generic-http-function-invocation.md - - page: gRPC Function Invocation - path: ../../docs/v0.6.1/grpc-function-invocation.md - - page: Container Functions - path: ../../docs/v0.6.1/container-functions.md - - page: Helm Functions - path: ../../docs/v0.6.1/helm-functions.md - - page: Streaming Functions - path: ../../docs/v0.6.1/streaming-functions.md - - page: Configure Autoscaling - path: ../../docs/v0.6.1/configure-autoscaling.md - - page: CLI - path: ../../docs/v0.6.1/cli.md - - page: Task Creation - path: ../../docs/v0.6.1/task-creation.md - - page: Container-Based Task Creation - path: ../../docs/v0.6.1/container-tasks.md - - page: Helm-Based Task Creation - path: ../../docs/v0.6.1/helm-tasks.md - - section: Function Autoscaling skip-slug: true contents: - page: Function Autoscaling Overview - path: ../../docs/v0.6.1/autoscaling/index.md + path: ../../../docs/v0.6.1/autoscaling/index.md - page: Architecture - path: ../../docs/v0.6.1/autoscaling/architecture.md + path: ../../../docs/v0.6.1/autoscaling/architecture.md - page: Operations - path: ../../docs/v0.6.1/autoscaling/operations.md + path: ../../../docs/v0.6.1/autoscaling/operations.md - page: Observability - path: ../../docs/v0.6.1/autoscaling/observability.md + path: ../../../docs/v0.6.1/autoscaling/observability.md slug: autoscaler-observability - section: Observability skip-slug: true contents: - page: Observability - path: ../../docs/v0.6.1/observability.md + path: ../../../docs/v0.6.1/observability.md - page: Example Dashboards - path: ../../docs/v0.6.1/example-dashboards.md + path: ../../../docs/v0.6.1/example-dashboards.md - section: Metrics skip-slug: true contents: - page: Metrics Overview - path: ../../docs/v0.6.1/metrics/metrics-index.md + path: ../../../docs/v0.6.1/metrics/metrics-index.md - page: Cassandra - path: ../../docs/v0.6.1/metrics/cassandra/metrics.md + path: ../../../docs/v0.6.1/metrics/cassandra/metrics.md - page: ESS - path: ../../docs/v0.6.1/metrics/ess/metrics.md + path: ../../../docs/v0.6.1/metrics/ess/metrics.md - page: Init Container - path: ../../docs/v0.6.1/metrics/init-container/metrics.md + path: ../../../docs/v0.6.1/metrics/init-container/metrics.md - page: Invocation Service - path: ../../docs/v0.6.1/metrics/invocation-service/metrics.md + path: ../../../docs/v0.6.1/metrics/invocation-service/metrics.md - page: LLM API Gateway - path: ../../docs/v0.6.1/metrics/llm-api-gateway/metrics.md + path: ../../../docs/v0.6.1/metrics/llm-api-gateway/metrics.md - page: LLM Function Invocation Metrics Report - path: ../../docs/v0.6.1/metrics/llm-function-invocation-path.md + path: ../../../docs/v0.6.1/metrics/llm-function-invocation-path.md - page: LLM Request Router - path: ../../docs/v0.6.1/metrics/llm-request-router/metrics.md + path: ../../../docs/v0.6.1/metrics/llm-request-router/metrics.md - page: NVCF API - path: ../../docs/v0.6.1/metrics/nvcf-api/metrics.md + path: ../../../docs/v0.6.1/metrics/nvcf-api/metrics.md - page: SIS/Spot - path: ../../docs/v0.6.1/metrics/sis-spot/metrics.md + path: ../../../docs/v0.6.1/metrics/sis-spot/metrics.md - page: State Metrics - path: ../../docs/v0.6.1/metrics/state-metrics/metrics.md + path: ../../../docs/v0.6.1/metrics/state-metrics/metrics.md - page: Utils Container - path: ../../docs/v0.6.1/metrics/utils-container/metrics.md + path: ../../../docs/v0.6.1/metrics/utils-container/metrics.md - page: Vault/OpenBao - path: ../../docs/v0.6.1/metrics/vault-openbao/metrics.md + path: ../../../docs/v0.6.1/metrics/vault-openbao/metrics.md - section: Operations skip-slug: true contents: - page: Control Plane Operations - path: ../../docs/v0.6.1/control-plane-operations.md + path: ../../../docs/v0.6.1/control-plane-operations.md - page: Cluster Monitoring - path: ../../docs/v0.6.1/cluster-management/monitoring.md + path: ../../../docs/v0.6.1/cluster-management/monitoring.md - page: Troubleshooting - path: ../../docs/v0.6.1/troubleshooting.md + path: ../../../docs/v0.6.1/troubleshooting.md - section: Runbooks skip-slug: true contents: - page: Runbooks - path: ../../docs/v0.6.1/runbooks/index.md + path: ../../../docs/v0.6.1/runbooks/index.md - page: Key Rotation - path: ../../docs/v0.6.1/runbooks/control-plane-key-rotation-mek.md + path: ../../../docs/v0.6.1/runbooks/control-plane-key-rotation-mek.md - section: Cache Runbooks skip-slug: true contents: - page: Caches Overview - path: ../../docs/v0.6.1/runbooks/caches.md + path: ../../../docs/v0.6.1/runbooks/caches.md - link: DDCS Cache Misses href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/cache-misses.html - link: DDCS Client Sharding @@ -235,40 +205,30 @@ navigation: skip-slug: true contents: - page: Cluster Reference - path: ../../docs/v0.6.1/cluster-management/reference.md - - page: gRPC Load Testing - path: ../../docs/v0.6.1/grpc-load-testing.md - - page: gRPC Load Test SLI Guide - path: ../../docs/v0.6.1/grpc-load-test-sli-guide.md - - page: HTTP Load Testing - path: ../../docs/v0.6.1/http-load-testing.md - - page: HTTP Load Test SLI Guide - path: ../../docs/v0.6.1/http-load-test-sli-guide.md - - page: HTTP Soak Testing - path: ../../docs/v0.6.1/http-soak-testing.md + path: ../../../docs/v0.6.1/cluster-management/reference.md - section: Development skip-slug: true contents: - page: Architecture Overview - path: ../../docs/dev/architecture.md + path: ../../../docs/dev/architecture.md - section: Local Development skip-slug: true contents: - page: Local Development - path: ../../docs/v0.6.1/local-development.md + path: ../../../docs/v0.6.1/local-development.md - page: Single-cluster (CLI) - path: ../../docs/v0.6.1/local-development/single-cluster-cli.md + path: ../../../docs/v0.6.1/local-development/single-cluster-cli.md - page: Single-cluster (Helmfile) - path: ../../docs/v0.6.1/local-development/single-cluster-helmfile.md + path: ../../../docs/v0.6.1/local-development/single-cluster-helmfile.md - page: Multi-cluster (CLI) - path: ../../docs/v0.6.1/local-development/multi-cluster-cli.md + path: ../../../docs/v0.6.1/local-development/multi-cluster-cli.md - page: Multi-cluster (Helmfile) - path: ../../docs/v0.6.1/local-development/multi-cluster-helmfile.md + path: ../../../docs/v0.6.1/local-development/multi-cluster-helmfile.md - page: Fake GPU Operator - path: ../../docs/v0.6.1/fake-gpu-operator.md + path: ../../../docs/v0.6.1/fake-gpu-operator.md - page: Release Process - path: ../../docs/dev/release-process.md + path: ../../../docs/dev/release-process.md - section: Managed (Legacy) skip-slug: true @@ -277,25 +237,25 @@ navigation: skip-slug: true contents: - page: NGC-Managed Clusters - path: ../../docs/ngc-managed/cluster-management/ngc-managed.md + path: ../../../docs/ngc-managed/cluster-management/ngc-managed.md - page: Helm-Managed Clusters - path: ../../docs/ngc-managed/cluster-management/helm-managed.md + path: ../../../docs/ngc-managed/cluster-management/helm-managed.md - page: Configuration - path: ../../docs/ngc-managed/cluster-management/configuration.md + path: ../../../docs/ngc-managed/cluster-management/configuration.md - page: Monitoring - path: ../../docs/ngc-managed/cluster-management/monitoring.md + path: ../../../docs/ngc-managed/cluster-management/monitoring.md - page: Reference - path: ../../docs/ngc-managed/cluster-management/reference.md + path: ../../../docs/ngc-managed/cluster-management/reference.md - page: Container Cache - path: ../../docs/ngc-managed/cluster-management/container-cache.md + path: ../../../docs/ngc-managed/cluster-management/container-cache.md - page: GXCache - path: ../../docs/ngc-managed/cluster-management/gxcache.md + path: ../../../docs/ngc-managed/cluster-management/gxcache.md - page: KAI Scheduler - path: ../../docs/ngc-managed/cluster-management/kai-scheduler.md + path: ../../../docs/ngc-managed/cluster-management/kai-scheduler.md - page: Function Lifecycle - path: ../../docs/ngc-managed/function-lifecycle.md + path: ../../../docs/ngc-managed/function-lifecycle.md - page: Service Keys - path: ../../docs/ngc-managed/service-keys.md + path: ../../../docs/ngc-managed/service-keys.md - page: Observability slug: managed-observability - path: ../../docs/ngc-managed/observability.md + path: ../../../docs/ngc-managed/observability.md diff --git a/fern/versions/cp-0.20.6-compute-0.4.4-obs-0.2.2.yml b/fern/versions/cp-0.20.6-compute-0.4.4-obs-0.2.2.yml deleted file mode 100644 index 9e504ff4d2..0000000000 --- a/fern/versions/cp-0.20.6-compute-0.4.4-obs-0.2.2.yml +++ /dev/null @@ -1,321 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -navigation: - # No-op docs preview trigger for Fern bot alias validation. - - page: Overview - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/index.md - - page: Quickstart - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/quickstart.md - - - section: Release Notes - skip-slug: true - contents: - - page: Release Notes Overview - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/release-notes/index.md - - page: "0.20.6 Release Notes" - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/release-notes/0.20.6.md - - page: "0.6.1 to 0.20.6 Upgrade" - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/release-notes/0.6.1-to-0.20.6-upgrade.md - - - section: Before You Deploy - skip-slug: true - contents: - - page: Infrastructure Sizing - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/infrastructure-sizing.md - - page: Manifest - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/manifest.md - - - section: Deployment - skip-slug: true - contents: - - page: Installation Overview - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/installation.md - - page: Image Mirroring - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/image-mirroring.md - - page: Helmfile Installation - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/helmfile-installation.md - - page: CSP End-to-End Example - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/csp-end-to-end-example-installation.md - - - section: GPU Cluster Setup - skip-slug: true - contents: - - page: GPU Cluster Setup - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/index.md - - page: Self-Managed Clusters - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/self-managed.md - - page: Nsight Profiling - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/nsight-profiling.md - - - section: Configuration - skip-slug: true - contents: - - page: NVCF UI - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/nvcf-ui.md - - page: Optional Enhancements - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/optional-enhancements.md - - page: Pod Disruption Budgets - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/pod-disruption-budgets.md - - page: gRPC Invocation Enablement - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/grpc-invocation-enablement.md - - page: LLM Function Enablement - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/llm-function-enablement.md - - page: LLM Request Router Load Balancing - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/llm-request-router-load-balancing.md - - page: Gateway Routing - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/gateway-routing.md - - page: Third-Party Registries - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/third-party-registries.md - - page: Registry Allowlist - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/registry-allowlist.md - - page: Cluster Configuration - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/configuration.md - - page: Multi-Tenancy - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/multi-tenancy.md - - page: KAI Scheduler - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/kai-scheduler.md - - page: Gang Scheduling - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/gang-scheduling.md - - page: Topology-Aware Scheduling - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/topology-aware-scheduling.md - - section: Low Latency Streaming - skip-slug: true - contents: - - page: LLS Installation - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/lls-installation.md - - section: NVCF Caches - skip-slug: true - contents: - - page: Helm Model Caching - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/model-cache.md - - page: Container Cache - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/container-cache.md - - page: GXCache - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/gxcache.md - - section: Physical Simulation Caches - skip-slug: true - contents: - - page: Simulation Caches - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/caches.md - - section: DDCS - skip-slug: true - contents: - - link: DDCS - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/ - - link: DDCS Configuration - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/configure.html - - link: DDCS Deployment - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/deploy.html - - link: DDCS TLS - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/tls.html - - section: UCC - skip-slug: true - contents: - - link: UCC - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/ - - link: UCC Configuration - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/configure.html - - link: UCC Deployment - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/deploy.html - - link: UCC TLS - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/tls.html - - - section: Using Cloud Functions - skip-slug: true - contents: - - page: API - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/api.md - - page: Function Creation - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/function-creation.md - - page: LLM Gateway - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/llm-gateway.md - - page: Generic HTTP Function Invocation - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/generic-http-function-invocation.md - - page: gRPC Function Invocation - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/grpc-function-invocation.md - - page: Container Functions - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/container-functions.md - - page: Helm Functions - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/helm-functions.md - - page: Streaming Functions - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/streaming-functions.md - - page: Configure Autoscaling - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/configure-autoscaling.md - - page: CLI - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cli.md - - page: Task Creation - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/task-creation.md - - page: Container-Based Task Creation - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/container-tasks.md - - page: Helm-Based Task Creation - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/helm-tasks.md - - - section: Function Autoscaling - skip-slug: true - contents: - - page: Function Autoscaling Overview - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/autoscaling/index.md - - page: Architecture - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/autoscaling/architecture.md - - page: Operations - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/autoscaling/operations.md - - page: Observability - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/autoscaling/observability.md - slug: autoscaler-observability - - - section: Observability - skip-slug: true - contents: - - page: Observability - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/observability.md - - page: Example Dashboards - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/example-dashboards.md - - section: Metrics - skip-slug: true - contents: - - page: Metrics Overview - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/metrics-index.md - - page: Cassandra - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/cassandra/metrics.md - - page: ESS - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/ess/metrics.md - - page: Init Container - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/init-container/metrics.md - - page: Invocation Service - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/invocation-service/metrics.md - - page: LLM API Gateway - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/llm-api-gateway/metrics.md - - page: LLM Function Invocation Metrics Report - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/llm-function-invocation-path.md - - page: LLM Request Router - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/llm-request-router/metrics.md - - page: NVCF API - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/nvcf-api/metrics.md - - page: SIS/Spot - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/sis-spot/metrics.md - - page: State Metrics - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/state-metrics/metrics.md - - page: Utils Container - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/utils-container/metrics.md - - page: Vault/OpenBao - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/metrics/vault-openbao/metrics.md - - - section: Operations - skip-slug: true - contents: - - page: Control Plane Operations - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/control-plane-operations.md - - page: Cluster Monitoring - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/monitoring.md - - page: Troubleshooting - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/troubleshooting.md - - - section: Runbooks - skip-slug: true - contents: - - page: Runbooks - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/index.md - - page: Key Rotation - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/control-plane-key-rotation-mek.md - - page: Transport TLS Rotation - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/transport-tls-rotation.md - - section: Cache Runbooks - skip-slug: true - contents: - - page: Caches Overview - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/caches.md - - link: DDCS Cache Misses - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/cache-misses.html - - link: DDCS Client Sharding - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/client-sharding.html - - link: DDCS Disk Exhaustion - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/disk-exhaustion.html - - link: DDCS Network Bottlenecks - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/network-bottlenecks.html - - link: DDCS RocksDB Corruption - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/rocksdb-corruption.html - - link: UCC Connection Saturation - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/connection-saturation.html - - link: UCC Data Disk Bandwidth - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/data-disk-bandwidth.html - - link: UCC Metadata Cache Undersizing - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/metadata-cache-undersizing.html - - link: UCC Network Bandwidth Saturation - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/network-bandwidth-saturation.html - - link: UCC Upstream S3 Connections - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/upstream-s3-connections.html - - page: Quota Management - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/runbooks/update-nvcf-account-quota-limits.md - - section: Reference - skip-slug: true - contents: - - page: Cluster Reference - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/cluster-management/reference.md - - page: gRPC Load Testing - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/grpc-load-testing.md - - page: gRPC Load Test SLI Guide - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/grpc-load-test-sli-guide.md - - page: HTTP Load Testing - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/http-load-testing.md - - page: HTTP Load Test SLI Guide - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/http-load-test-sli-guide.md - - page: HTTP Soak Testing - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/http-soak-testing.md - - - section: Development - skip-slug: true - contents: - - page: Architecture Overview - path: ../../docs/dev/architecture.md - - page: Storage-Agnostic Cache Architecture - path: ../../docs/dev/sdd-storage-agnostic-cache-architecture.md - - section: Local Development - skip-slug: true - contents: - - page: Local Development - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development.md - - page: Single-cluster (CLI) - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development/single-cluster-cli.md - - page: Single-cluster (Helmfile) - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development/single-cluster-helmfile.md - - page: Multi-cluster (CLI) - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development/multi-cluster-cli.md - - page: Multi-cluster (Helmfile) - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/local-development/multi-cluster-helmfile.md - - page: Fake GPU Operator - path: ../../docs/cp-0.20.6-compute-0.4.4-obs-0.2.2/fake-gpu-operator.md - - page: Release Process - path: ../../docs/dev/release-process.md - - - section: Managed (Legacy) - skip-slug: true - contents: - - section: Cluster Management - skip-slug: true - contents: - - page: NGC-Managed Clusters - path: ../../docs/ngc-managed/cluster-management/ngc-managed.md - - page: Helm-Managed Clusters - path: ../../docs/ngc-managed/cluster-management/helm-managed.md - - page: GitOps Cluster Registration - path: ../../docs/ngc-managed/cluster-management/gitops-registration.md - - page: Configuration - path: ../../docs/ngc-managed/cluster-management/configuration.md - - page: Monitoring - path: ../../docs/ngc-managed/cluster-management/monitoring.md - - page: Reference - path: ../../docs/ngc-managed/cluster-management/reference.md - - page: Container Cache - path: ../../docs/ngc-managed/cluster-management/container-cache.md - - page: GXCache - path: ../../docs/ngc-managed/cluster-management/gxcache.md - - page: KAI Scheduler - path: ../../docs/ngc-managed/cluster-management/kai-scheduler.md - - page: Function Lifecycle - path: ../../docs/ngc-managed/function-lifecycle.md - - page: Service Keys - path: ../../docs/ngc-managed/service-keys.md - - page: Observability - slug: managed-observability - path: ../../docs/ngc-managed/observability.md diff --git a/fern/versions/dev.yml b/fern/versions/dev.yml deleted file mode 100644 index 0ea6a53bec..0000000000 --- a/fern/versions/dev.yml +++ /dev/null @@ -1,317 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -navigation: - # No-op docs preview trigger for Fern bot alias validation. - - page: Overview - path: ../../docs/user/index.md - - page: Quickstart - path: ../../docs/user/quickstart.md - - - section: Release Notes - skip-slug: true - contents: - - page: Release Notes Overview - path: ../../docs/user/release-notes/index.md - - - section: Before You Deploy - skip-slug: true - contents: - - page: Infrastructure Sizing - path: ../../docs/user/infrastructure-sizing.md - - page: Manifest - path: ../../docs/user/manifest.md - - - section: Deployment - skip-slug: true - contents: - - page: Installation Overview - path: ../../docs/user/installation.md - - page: Image Mirroring - path: ../../docs/user/image-mirroring.md - - page: Helmfile Installation - path: ../../docs/user/helmfile-installation.md - - page: CSP End-to-End Example - path: ../../docs/user/csp-end-to-end-example-installation.md - - - section: GPU Cluster Setup - skip-slug: true - contents: - - page: GPU Cluster Setup - path: ../../docs/user/cluster-management/index.md - - page: Self-Managed Clusters - path: ../../docs/user/cluster-management/self-managed.md - - page: Nsight Profiling - path: ../../docs/user/cluster-management/nsight-profiling.md - - - section: Configuration - skip-slug: true - contents: - - page: NVCF UI - path: ../../docs/user/nvcf-ui.md - - page: Optional Enhancements - path: ../../docs/user/optional-enhancements.md - - page: Pod Disruption Budgets - path: ../../docs/user/pod-disruption-budgets.md - - page: gRPC Invocation Enablement - path: ../../docs/user/grpc-invocation-enablement.md - - page: LLM Function Enablement - path: ../../docs/user/llm-function-enablement.md - - page: LLM Request Router Load Balancing - path: ../../docs/user/llm-request-router-load-balancing.md - - page: Gateway Routing - path: ../../docs/user/gateway-routing.md - - page: Third-Party Registries - path: ../../docs/user/third-party-registries.md - - page: Registry Allowlist - path: ../../docs/user/registry-allowlist.md - - page: Cluster Configuration - path: ../../docs/user/cluster-management/configuration.md - - page: Multi-Tenancy - path: ../../docs/user/cluster-management/multi-tenancy.md - - page: KAI Scheduler - path: ../../docs/user/cluster-management/kai-scheduler.md - - page: Gang Scheduling - path: ../../docs/user/cluster-management/gang-scheduling.md - - page: Topology-Aware Scheduling - path: ../../docs/user/cluster-management/topology-aware-scheduling.md - - section: Low Latency Streaming - skip-slug: true - contents: - - page: LLS Installation - path: ../../docs/user/lls-installation.md - - section: NVCF Caches - skip-slug: true - contents: - - page: Helm Model Caching - path: ../../docs/user/cluster-management/model-cache.md - - page: Container Cache - path: ../../docs/user/cluster-management/container-cache.md - - page: GXCache - path: ../../docs/user/cluster-management/gxcache.md - - section: Physical Simulation Caches - skip-slug: true - contents: - - page: Simulation Caches - path: ../../docs/user/caches.md - - section: DDCS - skip-slug: true - contents: - - link: DDCS - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/ - - link: DDCS Configuration - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/configure.html - - link: DDCS Deployment - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/deploy.html - - link: DDCS TLS - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/tls.html - - section: UCC - skip-slug: true - contents: - - link: UCC - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/ - - link: UCC Configuration - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/configure.html - - link: UCC Deployment - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/deploy.html - - link: UCC TLS - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/tls.html - - - section: Using Cloud Functions - skip-slug: true - contents: - - page: API - path: ../../docs/user/api.md - - page: Function Creation - path: ../../docs/user/function-creation.md - - page: LLM Gateway - path: ../../docs/user/llm-gateway.md - - page: Generic HTTP Function Invocation - path: ../../docs/user/generic-http-function-invocation.md - - page: gRPC Function Invocation - path: ../../docs/user/grpc-function-invocation.md - - page: Container Functions - path: ../../docs/user/container-functions.md - - page: Helm Functions - path: ../../docs/user/helm-functions.md - - page: Streaming Functions - path: ../../docs/user/streaming-functions.md - - page: Configure Autoscaling - path: ../../docs/user/configure-autoscaling.md - - page: CLI - path: ../../docs/user/cli.md - - page: Task Creation - path: ../../docs/user/task-creation.md - - page: Container-Based Task Creation - path: ../../docs/user/container-tasks.md - - page: Helm-Based Task Creation - path: ../../docs/user/helm-tasks.md - - - section: Function Autoscaling - skip-slug: true - contents: - - page: Function Autoscaling Overview - path: ../../docs/user/autoscaling/index.md - - page: Architecture - path: ../../docs/user/autoscaling/architecture.md - - page: Operations - path: ../../docs/user/autoscaling/operations.md - - page: Observability - path: ../../docs/user/autoscaling/observability.md - slug: autoscaler-observability - - - section: Observability - skip-slug: true - contents: - - page: Observability - path: ../../docs/user/observability.md - - page: Example Dashboards - path: ../../docs/user/example-dashboards.md - - section: Metrics - skip-slug: true - contents: - - page: Metrics Overview - path: ../../docs/user/metrics/metrics-index.md - - page: Cassandra - path: ../../docs/user/metrics/cassandra/metrics.md - - page: ESS - path: ../../docs/user/metrics/ess/metrics.md - - page: Init Container - path: ../../docs/user/metrics/init-container/metrics.md - - page: Invocation Service - path: ../../docs/user/metrics/invocation-service/metrics.md - - page: LLM API Gateway - path: ../../docs/user/metrics/llm-api-gateway/metrics.md - - page: LLM Function Invocation Metrics Report - path: ../../docs/user/metrics/llm-function-invocation-path.md - - page: LLM Request Router - path: ../../docs/user/metrics/llm-request-router/metrics.md - - page: NVCF API - path: ../../docs/user/metrics/nvcf-api/metrics.md - - page: SIS/Spot - path: ../../docs/user/metrics/sis-spot/metrics.md - - page: State Metrics - path: ../../docs/user/metrics/state-metrics/metrics.md - - page: Utils Container - path: ../../docs/user/metrics/utils-container/metrics.md - - page: Vault/OpenBao - path: ../../docs/user/metrics/vault-openbao/metrics.md - - - section: Operations - skip-slug: true - contents: - - page: Control Plane Operations - path: ../../docs/user/control-plane-operations.md - - page: Cluster Monitoring - path: ../../docs/user/cluster-management/monitoring.md - - page: Troubleshooting - path: ../../docs/user/troubleshooting.md - - - section: Runbooks - skip-slug: true - contents: - - page: Runbooks - path: ../../docs/user/runbooks/index.md - - page: Key Rotation - path: ../../docs/user/runbooks/control-plane-key-rotation-mek.md - - page: Transport TLS Rotation - path: ../../docs/user/runbooks/transport-tls-rotation.md - - section: Cache Runbooks - skip-slug: true - contents: - - page: Caches Overview - path: ../../docs/user/runbooks/caches.md - - link: DDCS Cache Misses - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/cache-misses.html - - link: DDCS Client Sharding - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/client-sharding.html - - link: DDCS Disk Exhaustion - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/disk-exhaustion.html - - link: DDCS Network Bottlenecks - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/network-bottlenecks.html - - link: DDCS RocksDB Corruption - href: https://docs.omniverse.nvidia.com/ovcaches/ddcs/5.0/runbooks/rocksdb-corruption.html - - link: UCC Connection Saturation - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/connection-saturation.html - - link: UCC Data Disk Bandwidth - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/data-disk-bandwidth.html - - link: UCC Metadata Cache Undersizing - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/metadata-cache-undersizing.html - - link: UCC Network Bandwidth Saturation - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/network-bandwidth-saturation.html - - link: UCC Upstream S3 Connections - href: https://docs.omniverse.nvidia.com/ovcaches/ucc/3.0/runbooks/upstream-s3-connections.html - - page: Quota Management - path: ../../docs/user/runbooks/update-nvcf-account-quota-limits.md - - section: Reference - skip-slug: true - contents: - - page: Cluster Reference - path: ../../docs/user/cluster-management/reference.md - - page: gRPC Load Testing - path: ../../docs/user/grpc-load-testing.md - - page: gRPC Load Test SLI Guide - path: ../../docs/user/grpc-load-test-sli-guide.md - - page: HTTP Load Testing - path: ../../docs/user/http-load-testing.md - - page: HTTP Load Test SLI Guide - path: ../../docs/user/http-load-test-sli-guide.md - - page: HTTP Soak Testing - path: ../../docs/user/http-soak-testing.md - - - section: Development - skip-slug: true - contents: - - page: Architecture Overview - path: ../../docs/dev/architecture.md - - page: Storage-Agnostic Cache Architecture - path: ../../docs/dev/sdd-storage-agnostic-cache-architecture.md - - section: Local Development - skip-slug: true - contents: - - page: Local Development - path: ../../docs/user/local-development.md - - page: Single-cluster (CLI) - path: ../../docs/user/local-development/single-cluster-cli.md - - page: Single-cluster (Helmfile) - path: ../../docs/user/local-development/single-cluster-helmfile.md - - page: Multi-cluster (CLI) - path: ../../docs/user/local-development/multi-cluster-cli.md - - page: Multi-cluster (Helmfile) - path: ../../docs/user/local-development/multi-cluster-helmfile.md - - page: Fake GPU Operator - path: ../../docs/user/fake-gpu-operator.md - - page: Release Process - path: ../../docs/dev/release-process.md - - - section: Managed (Legacy) - skip-slug: true - contents: - - section: Cluster Management - skip-slug: true - contents: - - page: NGC-Managed Clusters - path: ../../docs/ngc-managed/cluster-management/ngc-managed.md - - page: Helm-Managed Clusters - path: ../../docs/ngc-managed/cluster-management/helm-managed.md - - page: GitOps Cluster Registration - path: ../../docs/ngc-managed/cluster-management/gitops-registration.md - - page: Configuration - path: ../../docs/ngc-managed/cluster-management/configuration.md - - page: Monitoring - path: ../../docs/ngc-managed/cluster-management/monitoring.md - - page: Reference - path: ../../docs/ngc-managed/cluster-management/reference.md - - page: Container Cache - path: ../../docs/ngc-managed/cluster-management/container-cache.md - - page: GXCache - path: ../../docs/ngc-managed/cluster-management/gxcache.md - - page: KAI Scheduler - path: ../../docs/ngc-managed/cluster-management/kai-scheduler.md - - page: Function Lifecycle - path: ../../docs/ngc-managed/function-lifecycle.md - - page: Service Keys - path: ../../docs/ngc-managed/service-keys.md - - page: Observability - slug: managed-observability - path: ../../docs/ngc-managed/observability.md diff --git a/src/clis/nvcf-cli/README.md b/src/clis/nvcf-cli/README.md index 29e42e127b..b319258455 100644 --- a/src/clis/nvcf-cli/README.md +++ b/src/clis/nvcf-cli/README.md @@ -895,7 +895,7 @@ curl -sS -X POST "https://llm.invocation.${INVOCATION_DOMAIN}/v1/embeddings" \ -d "{\"model\":\"${FUNCTION_ID}/${MODEL_NAME}\",\"input\":\"NVCF embeddings check\"}" ``` -For LLM Gateway endpoint behavior, routing, and session stickiness details, see [LLM Gateway](../../../docs/user/llm-gateway.md). +For LLM Gateway endpoint behavior, routing, and session stickiness details, see [LLM Gateway](../../../docs/overview/llm-gateway.md). **New Features:** @@ -1637,7 +1637,7 @@ Sample configs live under [`examples/`](./examples): `gpuSpecification.instanceType`. When `resultHandlingStrategy=UPLOAD`, `resultsLocation` becomes required and the user must supply an `NGC_API_KEY` secret with write privileges to that location. See the -[OpenAPI specification](../../../docs/user/api.md#openapi-specification) for +[OpenAPI specification](../../../docs/overview/api.md#openapi-specification) for the full field reference. --- diff --git a/src/clis/nvcf-cli/USAGE-GUIDE.md b/src/clis/nvcf-cli/USAGE-GUIDE.md index 492b455061..8f4189ab83 100644 --- a/src/clis/nvcf-cli/USAGE-GUIDE.md +++ b/src/clis/nvcf-cli/USAGE-GUIDE.md @@ -1021,7 +1021,7 @@ curl -sS -X POST "https://llm.invocation.${INVOCATION_DOMAIN}/v1/embeddings" \ -d "{\"model\":\"${FUNCTION_ID}/${MODEL_NAME}\",\"input\":\"NVCF embeddings check\"}" ``` -For LLM Gateway endpoint behavior, routing, and session stickiness details, see [LLM Gateway](../../../docs/user/llm-gateway.md). +For LLM Gateway endpoint behavior, routing, and session stickiness details, see [LLM Gateway](../../../docs/overview/llm-gateway.md). #### Sample Invocation JSON (`examples/invoke-function.json`) diff --git a/src/clis/nvcf-cli/internal/selfhosted/preflight.go b/src/clis/nvcf-cli/internal/selfhosted/preflight.go index e4dc5858a1..1d820eb91b 100644 --- a/src/clis/nvcf-cli/internal/selfhosted/preflight.go +++ b/src/clis/nvcf-cli/internal/selfhosted/preflight.go @@ -404,7 +404,7 @@ func computePlaneCheckCategory(rc RoleConfig) categorySpec { } // Required minimum inotify limits per -// docs/user/cluster-management/self-managed.md#node-inotify-limits. +// docs/compute-plane/cluster-management/self-managed.md#node-inotify-limits. // NVCA bootstrap fails with "too many open files" when these are too low, // which surfaces downstream as opaque errors like empty clusterGroups or // "Invalid GPU specified" on function deploy. diff --git a/src/libraries/rust/stargate/docs/README.md b/src/libraries/rust/stargate/docs/README.md index acff8ebe09..bde3814450 100644 --- a/src/libraries/rust/stargate/docs/README.md +++ b/src/libraries/rust/stargate/docs/README.md @@ -11,7 +11,7 @@ tests. This page is navigation, not a second architecture specification. | Run the local stack or make a first request | [Local quickstart](getting-started/local-quickstart.md) | `kustomize/`, `crates/mock-dynamo`, `crates/pylon` | [Pylon onboarding](operations/pylon-onboarding.md) | | Integrate a gateway or use the HTTP proxy | [API gateway contract](api-gateway-contract.md) | `crates/stargate/src/http_proxy.rs`, `crates/stargate/tests/suite/proxy_contract.rs` | [gRPC API](reference/grpc-api.md), [config and environment](reference/config-and-env.md) | | Change registration, routing, retries, or clusters | [Multi-backend cluster routing](multi-backend-clusters.md) | `crates/stargate/src/routing_state/`, `crates/stargate/src/load_balancer/`, lifecycle and proxy tests | [Feature and behavior tests](feature-behavior-test-matrix.md) | -| Configure load-balancing algorithms or request overrides | [Load balancer configuration](load-balancer-configuration.md) | `crates/stargate/src/load_balancer/`, `crates/stargate/src/http_proxy/` | [NVCF request-router metrics](../../../../../docs/user/metrics/llm-request-router/metrics.md), [API gateway contract](api-gateway-contract.md) | +| Configure load-balancing algorithms or request overrides | [Load balancer configuration](load-balancer-configuration.md) | `crates/stargate/src/load_balancer/`, `crates/stargate/src/http_proxy/` | [NVCF request-router metrics](../../../../../docs/observability/metrics/llm-request-router/metrics.md), [API gateway contract](api-gateway-contract.md) | | Add or onboard a pylon model | [Pylon onboarding](operations/pylon-onboarding.md) | `crates/pylon-lib/src/`, `crates/pylon/src/` | [Runtime stats interface](runtime-stats-interface.md), [CLI reference](reference/cli.md) | | Change QUIC, HTTP/3, WebTransport, or backend connectivity | [Tunnel transport selection](tunnel-transports.md) | `crates/protocol/`, `crates/pylon-lib/src/quic_http_tunnel/`, `crates/stargate-k8s-router/` | [Deployment shape](operations/deployment-shape.md) | | Deploy or operate Stargate and pylon in Kubernetes | [Deployment shape](operations/deployment-shape.md) | `kustomize/`, `scripts/run_k8s_integ.py`, `scripts/run_tilt.py` | [Pylon onboarding](operations/pylon-onboarding.md), [troubleshooting](operations/troubleshooting.md) | diff --git a/src/libraries/rust/stargate/docs/load-balancer-configuration.md b/src/libraries/rust/stargate/docs/load-balancer-configuration.md index cc8f597854..7bd93f80b9 100644 --- a/src/libraries/rust/stargate/docs/load-balancer-configuration.md +++ b/src/libraries/rust/stargate/docs/load-balancer-configuration.md @@ -460,7 +460,7 @@ for status, replay, and retry-budget rules. Stargate records algorithm and fallback choices, proxy attempts and retries, admission rejections, upstream latency, and active backend counts. The prefix is configurable with `--metrics-prefix`. See the -[NVCF request-router metrics reference](../../../../../docs/user/metrics/llm-request-router/metrics.md) +[NVCF request-router metrics reference](../../../../../docs/observability/metrics/llm-request-router/metrics.md) for metric names, labels, and descriptions. The proxy request span records the effective comparator in `routing.comparator`. diff --git a/tests/bdd/features/single-cluster-eks-helmfile.feature b/tests/bdd/features/single-cluster-eks-helmfile.feature index 62fdb26516..1b5075167d 100644 --- a/tests/bdd/features/single-cluster-eks-helmfile.feature +++ b/tests/bdd/features/single-cluster-eks-helmfile.feature @@ -71,7 +71,7 @@ Feature: Install a single-cluster NVCF stack on a pre-provisioned EKS cluster wi @gateway-setup Scenario: Install gateway, capture ELB address, and author the EKS env file # Captures the user's manual setup steps per - # docs/user/gateway-routing.md. Installs the + # docs/self-managed/gateway-routing.md. Installs the # envoy-gateway controller, applies the nvcf-gateway Gateway, # waits for AWS to provision the NLB, captures the assigned # hostname into EKS_GATEWAY_ADDR, and patches eks-bdd.yaml with diff --git a/tests/bdd/features/single-cluster-up-oneclick.feature b/tests/bdd/features/single-cluster-up-oneclick.feature index ba978743ed..357024b300 100644 --- a/tests/bdd/features/single-cluster-up-oneclick.feature +++ b/tests/bdd/features/single-cluster-up-oneclick.feature @@ -6,7 +6,7 @@ Feature: Bring up a local single-cluster NVCF stack with the self-hosted up one- validate the documented quickstart (nvcf-cli self-hosted up) end to end. # self-hosted up is the local k3d single-cluster one-click documented in - # docs/user/quickstart.md. It defaults to --env local and requires a k3d-* + # docs/overview/quickstart.md. It defaults to --env local and requires a k3d-* # context, then runs the full pipeline in one command: preflight, resolve # stack, install the control plane, mint the admin token + discover the # issuer, register the cluster, install the compute plane, and print a diff --git a/tools/ci/test-github-release.py b/tools/ci/test-github-release.py index 29e845221f..bfe8ce6ad9 100644 --- a/tools/ci/test-github-release.py +++ b/tools/ci/test-github-release.py @@ -1606,6 +1606,52 @@ def test_release_metadata_publishes_one_inventory_per_stack(self): self.assertIsNotNone(service) self.assertEqual(service["resolved_inventory_asset"], asset_name) + STACK_INVENTORY_PUBLISHERS = { + "self-managed": ("nvcf-self-managed-stack", "nvcf-self-managed-stack-inventory.json"), + "nvcf-compute-plane": ("nvcf-compute-plane-stack", "nvcf-compute-plane-stack-inventory.json"), + "observability": ("nvcf-observability-stack", "nvcf-observability-stack-inventory.json"), + } + + def test_release_branch_tags_keep_the_same_inventory_publisher(self): + # Inventory attachment is keyed by the tag alone. A tag cut from a + # release train branch must resolve exactly as one cut from main did + # before the stacks moved to release branching. + metadata = json.loads(SCRIPT_PATH.with_name("github-release-subprojects.json").read_text()) + root = SCRIPT_PATH.parents[2] + for stack, (service_id, asset_name) in self.STACK_INVENTORY_PUBLISHERS.items(): + tag = f"deploy/stacks/{stack}/v1.1.0" + with self.subTest(tag=tag): + service = self.github_release.release_asset_service(metadata, tag, root) + self.assertIsNotNone(service) + self.assertEqual(service["id"], service_id) + self.assertEqual(service["resolved_inventory_asset"], asset_name) + self.assertEqual(self.github_release.version_from_tag(service, tag, root), "1.1.0") + + def test_compute_plane_legacy_prefix_tag_still_resolves_its_inventory_publisher(self): + metadata = json.loads(SCRIPT_PATH.with_name("github-release-subprojects.json").read_text()) + root = SCRIPT_PATH.parents[2] + tag = "nvcf-compute-plane-stack-v0.2.0" + service = self.github_release.release_asset_service(metadata, tag, root) + self.assertIsNotNone(service) + self.assertEqual(service["id"], "nvcf-compute-plane-stack") + self.assertEqual(service["resolved_inventory_asset"], "nvcf-compute-plane-stack-inventory.json") + self.assertEqual(self.github_release.version_from_tag(service, tag, root), "0.2.0") + + def test_release_train_branch_name_round_trips_for_every_stack(self): + metadata = json.loads(SCRIPT_PATH.with_name("github-release-subprojects.json").read_text()) + root = SCRIPT_PATH.parents[2] + for stack, (service_id, _asset_name) in self.STACK_INVENTORY_PUBLISHERS.items(): + with self.subTest(stack=stack): + service = self.github_release.find_service(metadata, service_id) + branch = self.github_release.service_release_branch(service, "1.1.0", root) + self.assertEqual(branch, f"release-deploy/stacks/{stack}/v1.1") + self.assertEqual(self.github_release.release_branch_train(service, branch, root), "1.1") + self.assertEqual( + self.github_release.tag_for_version(service, "1.1.0", root), + f"deploy/stacks/{stack}/v1.1.0", + ) + self.assertTrue(service.get("release_branch_only")) + def chart_release_metadata(self): """Return minimal release metadata for the chart publication tests.""" return { diff --git a/tools/docs-version-sync/README.md b/tools/docs-version-sync/README.md index 86187a7f81..efb31c251e 100644 --- a/tools/docs-version-sync/README.md +++ b/tools/docs-version-sync/README.md @@ -1,30 +1,70 @@ # Documentation Version Sync This tool keeps top-of-tree documentation aligned with three released stack -inventories. It also promotes an exact QA-qualified three-stack release set to -versioned documentation. +inventories. It renders the cross-stack compatibility matrix and writes the +catalog snapshot that freezes one stack's documentation train. ## Release and documentation flow ```text -merge release-worthy stack change to main - -> release automation creates the owning stack tag and GitHub Release +merge stack change to main + -> a maintainer cuts or updates release-deploy/stacks//vX.Y + -> a push to that branch creates the stack tag and GitHub Release -> the tag workflow attaches that stack's inventory JSON -> a maintainer runs docs-version-sync -> the catalog records public locations or Publication pending - -> generated blocks under docs/user/ are updated in a Pull Request + -> generated blocks under the documentation product trees are updated ``` -The stack release is automatic. A merge to `main` runs +Stacks release only from release branches. `main` never cuts a stack version. +A push to `release-deploy/stacks//vX.Y` runs [`release-tags.yml`](../../.github/workflows/release-tags.yml), which invokes -`tools/ci/github-release auto` for every registered subproject. A `feat:`, -`fix:`, or `perf:` commit that changes a stack creates that stack's next tag -and GitHub Release. Chart pin and Helmfile changes should use a release-worthy -commit type. No maintainer normally creates the stack tag by hand. +`tools/ci/github-release auto` and cuts the next `X.Y.Z` tag when the push +changes something the stack ships. `deploy/stacks//VERSION` names the +next train. See [`RELEASE.md`](../../RELEASE.md) for branch cutting. The tag workflow renders only the states owned by the tagged stack. It attaches the resolved chart and image inventory before publishing the GitHub Release. -Public artifact publishing is a separate process and can happen later after QA. +Release attachment is keyed by tag, not by branch. Public artifact publishing +is a separate process and can happen later after QA. + +## Documentation product trees + +Documentation is split into one Fern product per stack plus a shared overview: + +| Tree | Product | Versioned | +| --- | --- | --- | +| `docs/overview/` | Overview (matrix, quickstart, manifest, image mirroring) | No | +| `docs/self-managed/` | Self-Managed Stack (control plane) | Yes | +| `docs/compute-plane/` | Compute Plane Stack | Yes | +| `docs/observability/` | Observability Stack | Yes | + +Generated blocks live only in these four trees. Frozen copies +(`docs/-/`) are never regenerated; the catalog rejects output +paths that point at them. + +## Compatibility matrix + +`docs/overview/compatibility-matrix.md` carries the `compatibility-matrix` +generated block. It renders the current release of each stack from +`release_set.stacks` and the declared `compatibility` entries: + +```yaml +compatibility: + - stack: observability + train: "1.1" + compatible_with: + control-plane: "1.0+" + compute-plane: "1.0+" +``` + +Each entry names one stack train and the minimum train of the other two +stacks it works with: `X.Y+` means that train or later, `X.Y` means that +train only. The example renders as "Observability 1.1 works with Self-managed +1.0 or later, Compute plane 1.0 or later". Stack names use the `release_set` keys +(`control-plane`, `compute-plane`, `observability`). Add an entry when a stack +opens a new train, and raise a minimum when a release stops working with an +older train of another stack. Then regenerate the documentation. Each registered stack publishes its own inventory: @@ -44,10 +84,10 @@ go run -C tools/docs-version-sync . --target main --update-catalog ``` The command selects the latest stable release for all three stacks and records -a development release set. It updates: +each stack as development documentation. It updates: - `docs/version-catalog/main.yaml` -- Generated blocks configured by the catalog under `docs/user/` +- Generated blocks configured by the catalog under the product trees - The self-managed, compute-plane, and observability bundle versions - The exact source tag, commit, and inventory asset for all three stacks @@ -100,28 +140,36 @@ go run -C tools/docs-version-sync . --target main A publication-only update does not require a new stack release. -## Promote a QA-qualified release set +## Freeze a stack documentation train -After QA approves the three stack versions and all artifacts are published, -run the catalog update with all three exact versions: +Each stack freezes documentation on its own schedule. No joint qualification +of all three stacks is required. After QA approves a stack release on train +`X.Y` and its artifacts are published: ```bash -go run -C tools/docs-version-sync . \ - --target main \ - --update-catalog \ - --qualification-version cp-A.B.C-compute-D.E.F-obs-G.H.I \ - --stack-version A.B.C \ - --compute-stack-version D.E.F \ - --observability-stack-version G.H.I +git fetch --tags origin +go run -C tools/docs-version-sync . --target main --update-catalog go run -C tools/docs-version-sync . --target main -./tools/scripts/cut-docs-version.sh cp-A.B.C-compute-D.E.F-obs-G.H.I +./tools/scripts/cut-docs-version.sh --stack observability --train X.Y ``` -The qualification command does not select latest versions. The docs snapshot -fails unless the catalog is qualified and its documentation version matches the -requested version. The documentation version identifies all three stack -versions in `cp-X.Y.Z-compute-X.Y.Z-obs-X.Y.Z` format. The version dropdown -also labels each stack version. +The cut script runs `--freeze-stack --freeze-train X.Y`, +which checks that the stack's current release in `release_set.stacks` belongs +to train `X.Y`, warns when `publication_pending` is non-empty (the frozen +manifest keeps the pending markers), and writes +`docs/version-catalog/-X.Y.yaml` with that one stack marked +`qualified`. `main.yaml` stays in development state. The script then copies +`docs//` to `docs/-X.Y/`, generates +`fern/products//X.Y.yml`, and prints the `versions:` entry to add to +`fern/docs.yml`. + +Stack names for `--freeze-stack` are `control-plane`, `compute-plane`, and +`observability`. The script accepts the product slugs `self-managed`, +`compute-plane`, and `observability` and maps them. Overview documentation is +unversioned and is never cut. + +Add or adjust `compatibility` entries for the new train before regenerating so +the matrix reflects the qualified combination. ## Add an artifact to the stack inventory @@ -141,8 +189,8 @@ For a chart or image deployed by any stack: - If an independently released chart must be rendered from its immutable GitHub tag, add it to owning `release-inventory.yaml`. -3. Merge the release-worthy change to `main`. Release automation creates the - owning stack release. +3. Merge the change to `main`, then land it on the owning stack's release + branch. A push to the release branch creates the stack release. 4. Run the documentation sync after all selected stack releases have inventory assets. 5. Add a `manifest.entries` record for the new artifact description and source. diff --git a/tools/docs-version-sync/catalog.go b/tools/docs-version-sync/catalog.go index 7c8a50ae0d..e68c0272b6 100644 --- a/tools/docs-version-sync/catalog.go +++ b/tools/docs-version-sync/catalog.go @@ -31,18 +31,22 @@ const ( controlStackResourceName = "nvcf-self-managed-stack" computeStackResourceName = "nvcf-compute-plane-stack" observabilityStackResourceName = "nvcf-observability-stack" - releaseSetVersionFormat = "cp-X.Y.Z-compute-X.Y.Z-obs-X.Y.Z" + documentationTrainFormat = "X.Y" defaultStackRegistry = "public-resources" defaultImageRegistry = "public-images" defaultChartRegistry = "public-helm" ) var ( - fullLowercaseCommitSHARe = regexp.MustCompile(`^[0-9a-f]{40}$`) - lowercaseSHA256DigestRe = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) - releaseSetDocumentationVersionRe = regexp.MustCompile(`^cp-((?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*))-compute-((?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*))-obs-((?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*))$`) + fullLowercaseCommitSHARe = regexp.MustCompile(`^[0-9a-f]{40}$`) + lowercaseSHA256DigestRe = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + documentationTrainRe = regexp.MustCompile(`^(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)$`) ) +// documentationProductTrees are the only documentation trees that carry generated blocks. +// Frozen copies (docs/-/) are never regenerated. +var documentationProductTrees = []string{"docs/overview/", "docs/self-managed/", "docs/compute-plane/", "docs/observability/"} + type ArtifactType string const ( @@ -88,19 +92,20 @@ type ManifestEntry struct { } type Catalog struct { - Version int `yaml:"version"` - Target string `yaml:"target"` - Registries map[string]Registry `yaml:"registries"` - Publications []Publication `yaml:"publications,omitempty"` - VersionOverrides []VersionOverride `yaml:"version_overrides,omitempty"` - PublicationPending []string `yaml:"publication_pending,omitempty"` - Manifest ManifestMetadata `yaml:"manifest,omitempty"` - ReleaseSet ReleaseSetMetadata `yaml:"release_set,omitempty"` - Stack StackMetadata `yaml:"stack"` - Denylist []DenylistEntry `yaml:"denylist,omitempty"` - Artifacts []Artifact `yaml:"artifacts"` - SupplementalArtifacts []Artifact `yaml:"supplemental_artifacts"` - Outputs []OutputFile `yaml:"outputs"` + Version int `yaml:"version"` + Target string `yaml:"target"` + Registries map[string]Registry `yaml:"registries"` + Publications []Publication `yaml:"publications,omitempty"` + VersionOverrides []VersionOverride `yaml:"version_overrides,omitempty"` + PublicationPending []string `yaml:"publication_pending,omitempty"` + Manifest ManifestMetadata `yaml:"manifest,omitempty"` + ReleaseSet ReleaseSetMetadata `yaml:"release_set,omitempty"` + Compatibility []CompatibilityEntry `yaml:"compatibility,omitempty"` + Stack StackMetadata `yaml:"stack"` + Denylist []DenylistEntry `yaml:"denylist,omitempty"` + Artifacts []Artifact `yaml:"artifacts"` + SupplementalArtifacts []Artifact `yaml:"supplemental_artifacts"` + Outputs []OutputFile `yaml:"outputs"` } type ReleaseSetStatus string @@ -111,10 +116,10 @@ const ( ) // ReleaseSetMetadata identifies the three stack releases represented by the catalog. +// Each stack carries its own documentation version because the stacks release +// and freeze documentation independently. type ReleaseSetMetadata struct { - DocumentationVersion string `yaml:"documentation_version"` - Status ReleaseSetStatus `yaml:"status"` - Stacks ReleaseSetStacks `yaml:"stacks"` + Stacks ReleaseSetStacks `yaml:"stacks"` } type ReleaseSetStacks struct { @@ -124,10 +129,78 @@ type ReleaseSetStacks struct { } type StackReleaseMetadata struct { - Version string `yaml:"version"` - SourceTag string `yaml:"source_tag"` - SourceCommit string `yaml:"source_commit"` - InventoryAsset string `yaml:"inventory_asset"` + Version string `yaml:"version"` + SourceTag string `yaml:"source_tag"` + SourceCommit string `yaml:"source_commit"` + InventoryAsset string `yaml:"inventory_asset"` + DocumentationVersion string `yaml:"documentation_version"` + Status ReleaseSetStatus `yaml:"status"` +} + +// CompatibilityEntry declares the minimum train of each other stack that one +// train of a stack works with. Keys use the release_set stack names. Values +// are "X.Y+" (that train or later) or "X.Y" (that train only). +type CompatibilityEntry struct { + Stack string `yaml:"stack"` + Train string `yaml:"train"` + CompatibleWith map[string]string `yaml:"compatible_with"` +} + +const ( + releaseSetStackControlPlane = "control-plane" + releaseSetStackComputePlane = "compute-plane" + releaseSetStackObservability = "observability" +) + +var releaseSetStackNames = []string{releaseSetStackControlPlane, releaseSetStackComputePlane, releaseSetStackObservability} + +// documentationProductSlug maps a release_set stack name to its Fern product slug. +func documentationProductSlug(stack string) (string, error) { + switch stack { + case releaseSetStackControlPlane: + return "self-managed", nil + case releaseSetStackComputePlane, releaseSetStackObservability: + return stack, nil + default: + return "", fmt.Errorf("unknown release_set stack %q; want %s", stack, strings.Join(releaseSetStackNames, ", ")) + } +} + +func documentationStackDisplayName(stack string) string { + switch stack { + case releaseSetStackControlPlane: + return "Self-managed (control plane)" + case releaseSetStackComputePlane: + return "Compute plane" + case releaseSetStackObservability: + return "Observability" + } + return stack +} + +func (stacks *ReleaseSetStacks) byName(stack string) (*StackReleaseMetadata, error) { + switch stack { + case releaseSetStackControlPlane: + return &stacks.ControlPlane, nil + case releaseSetStackComputePlane: + return &stacks.ComputePlane, nil + case releaseSetStackObservability: + return &stacks.Observability, nil + default: + return nil, fmt.Errorf("unknown release_set stack %q; want %s", stack, strings.Join(releaseSetStackNames, ", ")) + } +} + +// releaseTrain returns the X.Y train of a semantic version. +func releaseTrain(version string) (string, bool) { + if !validStackVersion(version) { + return "", false + } + parts := strings.SplitN(version, ".", 3) + if len(parts) < 3 { + return "", false + } + return parts[0] + "." + parts[1], true } type Registry struct { @@ -359,6 +432,9 @@ func ValidateCatalog(catalog *Catalog) error { return err } } + if err := validateCompatibility(catalog.Compatibility); err != nil { + return err + } if strings.TrimSpace(catalog.Stack.Name) == "" { return fmt.Errorf("stack name cannot be empty") } @@ -464,33 +540,14 @@ func ValidateCatalog(catalog *Catalog) error { } func validateReleaseSet(releaseSet ReleaseSetMetadata) error { - if releaseSet.DocumentationVersion == "" || releaseSet.DocumentationVersion != strings.TrimSpace(releaseSet.DocumentationVersion) { - return fmt.Errorf("release_set documentation_version must be non-empty and trimmed") - } - if releaseSet.Status != ReleaseSetDevelopment && releaseSet.Status != ReleaseSetQualified { - return fmt.Errorf("release_set status must be development or qualified") - } - if releaseSet.Status == ReleaseSetDevelopment && releaseSet.DocumentationVersion != "dev" { - return fmt.Errorf("development release_set documentation_version must be dev") - } - if releaseSet.Status == ReleaseSetQualified { - _, _, _, ok := parseReleaseSetDocumentationVersion(releaseSet.DocumentationVersion) - if !ok { - return fmt.Errorf("qualified release_set documentation_version must use %s", releaseSetVersionFormat) - } - expected := releaseSetDocumentationVersion(releaseSet.Stacks) - if releaseSet.DocumentationVersion != expected { - return fmt.Errorf("qualified release_set documentation_version must be %s", expected) - } - } for _, stack := range []struct { name string metadata StackReleaseMetadata key string }{ - {name: "control-plane", metadata: releaseSet.Stacks.ControlPlane, key: selfManagedStackKey}, - {name: "compute-plane", metadata: releaseSet.Stacks.ComputePlane, key: computePlaneStackKey}, - {name: "observability", metadata: releaseSet.Stacks.Observability, key: observabilityStackKey}, + {name: releaseSetStackControlPlane, metadata: releaseSet.Stacks.ControlPlane, key: selfManagedStackKey}, + {name: releaseSetStackComputePlane, metadata: releaseSet.Stacks.ComputePlane, key: computePlaneStackKey}, + {name: releaseSetStackObservability, metadata: releaseSet.Stacks.Observability, key: observabilityStackKey}, } { spec, err := stackInventorySpecByKey(stack.key) if err != nil { @@ -508,27 +565,72 @@ func validateReleaseSet(releaseSet ReleaseSetMetadata) error { if stack.metadata.InventoryAsset != spec.AssetName { return fmt.Errorf("release_set %s inventory_asset must be %s", stack.name, spec.AssetName) } + if err := validateStackDocumentationVersion(stack.name, stack.metadata); err != nil { + return err + } } return nil } -func parseReleaseSetDocumentationVersion(version string) (string, string, string, bool) { - match := releaseSetDocumentationVersionRe.FindStringSubmatch(version) - if match == nil { - return "", "", "", false +func validateStackDocumentationVersion(name string, metadata StackReleaseMetadata) error { + if metadata.DocumentationVersion == "" || metadata.DocumentationVersion != strings.TrimSpace(metadata.DocumentationVersion) { + return fmt.Errorf("release_set %s documentation_version must be non-empty and trimmed", name) + } + switch metadata.Status { + case ReleaseSetDevelopment: + if metadata.DocumentationVersion != "dev" { + return fmt.Errorf("development release_set %s documentation_version must be dev", name) + } + case ReleaseSetQualified: + if !documentationTrainRe.MatchString(metadata.DocumentationVersion) { + return fmt.Errorf("qualified release_set %s documentation_version must use %s", name, documentationTrainFormat) + } + train, _ := releaseTrain(metadata.Version) + if metadata.DocumentationVersion != train { + return fmt.Errorf("qualified release_set %s documentation_version must be %s for version %s", name, train, metadata.Version) + } + default: + return fmt.Errorf("release_set %s status must be development or qualified", name) } - return match[1], match[2], match[3], true + return nil } -func releaseSetDocumentationVersion(stacks ReleaseSetStacks) string { - return fmt.Sprintf( - "cp-%s-compute-%s-obs-%s", - stacks.ControlPlane.Version, - stacks.ComputePlane.Version, - stacks.Observability.Version, - ) +func validateCompatibility(entries []CompatibilityEntry) error { + seen := map[string]struct{}{} + for _, entry := range entries { + if _, err := documentationProductSlug(entry.Stack); err != nil { + return fmt.Errorf("compatibility: %w", err) + } + if !documentationTrainRe.MatchString(entry.Train) { + return fmt.Errorf("compatibility %s train %q must use %s", entry.Stack, entry.Train, documentationTrainFormat) + } + key := entry.Stack + "\x00" + entry.Train + if _, exists := seen[key]; exists { + return fmt.Errorf("duplicate compatibility entry for %s %s", entry.Stack, entry.Train) + } + seen[key] = struct{}{} + if len(entry.CompatibleWith) != len(releaseSetStackNames)-1 { + return fmt.Errorf("compatibility %s %s must list exactly the other %d stacks", entry.Stack, entry.Train, len(releaseSetStackNames)-1) + } + for other, requirement := range entry.CompatibleWith { + if _, err := documentationProductSlug(other); err != nil { + return fmt.Errorf("compatibility %s %s: %w", entry.Stack, entry.Train, err) + } + if other == entry.Stack { + return fmt.Errorf("compatibility %s %s cannot list its own stack", entry.Stack, entry.Train) + } + if !compatibilityRequirementRe.MatchString(requirement) { + return fmt.Errorf("compatibility %s %s: %s requirement %q must use %s", entry.Stack, entry.Train, other, requirement, compatibilityRequirementFormat) + } + } + } + return nil } +const compatibilityRequirementFormat = "X.Y+ (that train or later) or X.Y (that train only)" + +var compatibilityRequirementRe = regexp.MustCompile(`^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\+?$`) + func validateManifestMetadata(metadata ManifestMetadata) error { seen := map[string]struct{}{} // Changing the EA-CVE allowlist requires an explicit documentation decision. @@ -661,13 +763,15 @@ func validateOutputPath(path string) error { if filepath.IsAbs(path) || strings.HasPrefix(clean, "../") || clean == ".." { return fmt.Errorf("output path %s must be repository relative", path) } - if strings.HasPrefix(clean, "docs/v0.5/") { - return fmt.Errorf("output path %s is not allowed: versioned docs are generated from main", path) + for _, tree := range documentationProductTrees { + if strings.HasPrefix(clean, tree) { + return nil + } } - if !strings.HasPrefix(clean, "docs/user/") { - return fmt.Errorf("output path %s is outside docs/user", path) + if strings.HasPrefix(clean, "docs/") { + return fmt.Errorf("output path %s is not allowed: frozen docs are generated from the product trees", path) } - return nil + return fmt.Errorf("output path %s is outside the documentation product trees %s", path, strings.Join(documentationProductTrees, ", ")) } func (catalog *Catalog) DenylistMap() map[string]DenylistEntry { @@ -844,6 +948,7 @@ func refreshCatalogFromArtifacts(stackVersion string, artifacts []Artifact, base catalog.PublicationPending = append(catalog.PublicationPending, base.PublicationPending...) catalog.Denylist = append(catalog.Denylist, base.Denylist...) catalog.Manifest = base.Manifest + catalog.Compatibility = append(catalog.Compatibility, base.Compatibility...) resolvedArtifacts := make(map[string]struct{}, len(catalog.Artifacts)) for _, artifact := range catalog.Artifacts { resolvedArtifacts[artifactNameAndTypeKey(artifact)] = struct{}{} @@ -961,14 +1066,21 @@ func (catalog *Catalog) pruneUnusedRegistries() { func defaultOutputs() []OutputFile { return []OutputFile{ { - Path: "docs/user/manifest.md", + Path: "docs/overview/compatibility-matrix.md", + Blocks: []OutputBlock{{ + Marker: "compatibility-matrix", + Renderer: "compatibility-matrix", + }}, + }, + { + Path: "docs/overview/manifest.md", Blocks: []OutputBlock{{ Marker: "manifest-artifact-registry-paths", Renderer: "manifest-artifact-registry-paths", }}, }, { - Path: "docs/user/image-mirroring.md", + Path: "docs/overview/image-mirroring.md", Blocks: []OutputBlock{ { Marker: "image-mirroring-resource-examples", @@ -992,7 +1104,7 @@ func defaultOutputs() []OutputFile { }, }, }, - {Path: "docs/user/cluster-management/self-managed.md"}, - {Path: "docs/user/cluster-management/reference.md"}, + {Path: "docs/compute-plane/cluster-management/self-managed.md"}, + {Path: "docs/compute-plane/cluster-management/reference.md"}, } } diff --git a/tools/docs-version-sync/catalog_inventory.go b/tools/docs-version-sync/catalog_inventory.go index 7ed0b07e5c..10ba3e6f23 100644 --- a/tools/docs-version-sync/catalog_inventory.go +++ b/tools/docs-version-sync/catalog_inventory.go @@ -28,7 +28,7 @@ func updateCatalogFromGitHub(repoRoot, sourceRef string, base *Catalog) (*Catalo return buildCatalogFromResolvedStackInventory(inventory, snapshot, base) } -func updateCatalogFromGitHubInventories(repoRoot string, sourceRefs map[string]string, qualificationVersion string, base *Catalog) (*Catalog, error) { +func updateCatalogFromGitHubInventories(repoRoot string, sourceRefs map[string]string, base *Catalog) (*Catalog, error) { client := newGitHubClientFromEnvironment() inventories := make(map[string]resolvedStackInventory, len(stackInventorySpecs)) for _, spec := range stackInventorySpecs { @@ -63,19 +63,14 @@ func updateCatalogFromGitHubInventories(repoRoot string, sourceRefs map[string]s if err != nil { return nil, err } - status := ReleaseSetDevelopment - documentationVersion := "dev" - if qualificationVersion != "" { - status = ReleaseSetQualified - documentationVersion = qualificationVersion - } - releaseSet, err := releaseSetFromInventories(inventories, documentationVersion, status) + releaseSet, err := releaseSetFromInventories(inventories) if err != nil { return nil, err } - if qualificationVersion == "" && base != nil && base.ReleaseSet.Status == ReleaseSetQualified && releaseSet.sameStackReleases(base.ReleaseSet) { - releaseSet.DocumentationVersion = base.ReleaseSet.DocumentationVersion - releaseSet.Status = base.ReleaseSet.Status + if base != nil { + if err := preserveQualifiedDocumentation(&releaseSet, base.ReleaseSet); err != nil { + return nil, err + } } catalog.ReleaseSet = releaseSet for _, spec := range stackInventorySpecs[1:] { @@ -96,9 +91,6 @@ func updateCatalogFromGitHubInventories(repoRoot string, sourceRefs map[string]s catalog.markAllUnpublishedAsPending() catalog.reconcilePublicationPending() catalog.pruneUnusedRegistries() - if qualificationVersion != "" && len(catalog.PublicationPending) > 0 { - return nil, fmt.Errorf("qualified release set has unpublished artifacts: %s", strings.Join(catalog.PublicationPending, ", ")) - } if err := ValidateCatalog(catalog); err != nil { return nil, err } diff --git a/tools/docs-version-sync/catalog_inventory_test.go b/tools/docs-version-sync/catalog_inventory_test.go index be9f97a857..b061687cb5 100644 --- a/tools/docs-version-sync/catalog_inventory_test.go +++ b/tools/docs-version-sync/catalog_inventory_test.go @@ -129,15 +129,21 @@ func TestUpdateCatalogFromGitHubInventoriesMergesThreeStackAssets(t *testing.T) selfManagedStackKey: releases[selfManagedStackKey].Version, computePlaneStackKey: releases[computePlaneStackKey].Version, observabilityStackKey: releases[observabilityStackKey].Version, - }, "", testCatalog()) + }, testCatalog()) if err != nil { t.Fatal(err) } if catalog.Stack.Version != "1.2.3" { t.Fatalf("self-managed stack version = %s, want 1.2.3", catalog.Stack.Version) } - if catalog.ReleaseSet.Status != ReleaseSetDevelopment || catalog.ReleaseSet.DocumentationVersion != "dev" { - t.Fatalf("release set = %#v, want development docs", catalog.ReleaseSet) + for _, stack := range []StackReleaseMetadata{ + catalog.ReleaseSet.Stacks.ControlPlane, + catalog.ReleaseSet.Stacks.ComputePlane, + catalog.ReleaseSet.Stacks.Observability, + } { + if stack.Status != ReleaseSetDevelopment || stack.DocumentationVersion != "dev" { + t.Fatalf("release set stack = %#v, want development docs", stack) + } } if catalog.ReleaseSet.Stacks.ControlPlane.Version != "1.2.3" || catalog.ReleaseSet.Stacks.ComputePlane.Version != "2.3.4" || diff --git a/tools/docs-version-sync/compatibility.go b/tools/docs-version-sync/compatibility.go new file mode 100644 index 0000000000..a9f2c070f7 --- /dev/null +++ b/tools/docs-version-sync/compatibility.go @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "sort" + "strings" +) + +// renderCompatibilityMatrix renders the current release of each stack and, for +// each stack train, the minimum train of the other stacks it works with. +func renderCompatibilityMatrix(catalog *Catalog) (string, error) { + if catalog.ReleaseSet == (ReleaseSetMetadata{}) { + return "", fmt.Errorf("compatibility matrix requires release_set stack releases") + } + if len(catalog.Compatibility) == 0 { + return "", fmt.Errorf("compatibility matrix requires at least one compatibility entry") + } + var b strings.Builder + b.WriteString("## Current stack releases\n\n") + b.WriteString("| Stack | Latest release | Source tag |\n| --- | --- | --- |\n") + for _, stack := range releaseSetStackNames { + metadata, err := catalog.ReleaseSet.Stacks.byName(stack) + if err != nil { + return "", err + } + slug, err := documentationProductSlug(stack) + if err != nil { + return "", err + } + b.WriteString(fmt.Sprintf("| [%s](/nvcf/%s/) | `%s` | `%s` |\n", + documentationStackDisplayName(stack), slug, metadata.Version, metadata.SourceTag)) + } + b.WriteString("\n## Compatible stack versions\n\n") + b.WriteString("| Stack | Release | Works with |\n| --- | --- | --- |\n") + entries := append([]CompatibilityEntry(nil), catalog.Compatibility...) + sort.SliceStable(entries, func(i, j int) bool { + if entries[i].Stack != entries[j].Stack { + return stackOrder(entries[i].Stack) < stackOrder(entries[j].Stack) + } + return compareTrains(entries[i].Train, entries[j].Train) > 0 + }) + for _, entry := range entries { + worksWith := make([]string, 0, len(releaseSetStackNames)-1) + for _, other := range releaseSetStackNames { + if other == entry.Stack { + continue + } + worksWith = append(worksWith, formatRequirement(other, entry.CompatibleWith[other])) + } + b.WriteString(fmt.Sprintf("| %s | `%s` | %s |\n", documentationStackDisplayName(entry.Stack), entry.Train, strings.Join(worksWith, ", "))) + } + return b.String(), nil +} + +func stackOrder(stack string) int { + for index, name := range releaseSetStackNames { + if name == stack { + return index + } + } + return len(releaseSetStackNames) +} + +// formatRequirement turns "1.0+" into "Compute plane 1.0 or later" and "1.0" +// into "Compute plane 1.0 only". +func formatRequirement(stack, requirement string) string { + name := documentationStackDisplayName(stack) + if train, open := strings.CutSuffix(requirement, "+"); open { + return fmt.Sprintf("%s `%s` or later", name, train) + } + return fmt.Sprintf("%s `%s` only", name, requirement) +} + +// compareTrains orders X.Y trains numerically; inputs are validated by the catalog. +func compareTrains(a, b string) int { + var aMajor, aMinor, bMajor, bMinor int + fmt.Sscanf(a, "%d.%d", &aMajor, &aMinor) + fmt.Sscanf(b, "%d.%d", &bMajor, &bMinor) + switch { + case aMajor != bMajor: + return aMajor - bMajor + default: + return aMinor - bMinor + } +} diff --git a/tools/docs-version-sync/docs-version-sync b/tools/docs-version-sync/docs-version-sync new file mode 100755 index 0000000000..fbbf471a99 Binary files /dev/null and b/tools/docs-version-sync/docs-version-sync differ diff --git a/tools/docs-version-sync/freeze.go b/tools/docs-version-sync/freeze.go new file mode 100644 index 0000000000..46dfbe80bd --- /dev/null +++ b/tools/docs-version-sync/freeze.go @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// freezeStackDocumentation writes the catalog snapshot that accompanies a +// frozen documentation train for one stack. The main catalog is left in +// development state; only the snapshot marks the stack qualified. +func freezeStackDocumentation(repoRoot, catalogPath, stack, train string) (string, error) { + slug, err := documentationProductSlug(stack) + if err != nil { + return "", err + } + if !documentationTrainRe.MatchString(train) { + return "", fmt.Errorf("--freeze-train must use %s", documentationTrainFormat) + } + catalog, err := LoadCatalog(catalogPath) + if err != nil { + return "", err + } + metadata, err := catalog.ReleaseSet.Stacks.byName(stack) + if err != nil { + return "", err + } + releasedTrain, ok := releaseTrain(metadata.Version) + if !ok { + return "", fmt.Errorf("release_set %s version %q is not a semantic version", stack, metadata.Version) + } + if releasedTrain != train { + return "", fmt.Errorf("release_set %s version %s belongs to train %s, not %s; sync the catalog to a %s.x release first", stack, metadata.Version, releasedTrain, train, train) + } + if len(catalog.PublicationPending) > 0 { + fmt.Fprintf(os.Stderr, "WARNING: freezing %s documentation while artifacts are publication pending: %s\n", stack, strings.Join(catalog.PublicationPending, ", ")) + } + if err := validateStackSourceSnapshot(repoRoot, catalog); err != nil { + return "", fmt.Errorf("validate stack source snapshot: %w", err) + } + metadata.DocumentationVersion = train + metadata.Status = ReleaseSetQualified + + snapshotPath := filepath.Join(filepath.Dir(catalogPath), slug+"-"+train+".yaml") + if _, err := os.Stat(snapshotPath); err == nil { + return "", fmt.Errorf("refusing to overwrite existing catalog snapshot %s", relOrAbs(repoRoot, snapshotPath)) + } else if !os.IsNotExist(err) { + return "", err + } + if err := WriteCatalog(snapshotPath, catalog); err != nil { + return "", err + } + return snapshotPath, nil +} diff --git a/tools/docs-version-sync/inline.go b/tools/docs-version-sync/inline.go index 820ca01689..c06ca88648 100644 --- a/tools/docs-version-sync/inline.go +++ b/tools/docs-version-sync/inline.go @@ -22,11 +22,11 @@ import ( func SyncInlineVersions(path, content string, catalog *Catalog) (string, bool, error) { switch path { - case "docs/user/image-mirroring.md": + case "docs/overview/image-mirroring.md": return syncImageMirroring(content, catalog) - case "docs/user/cluster-management/self-managed.md": + case "docs/compute-plane/cluster-management/self-managed.md": return syncClusterManagementSelfManaged(content, catalog) - case "docs/user/cluster-management/reference.md": + case "docs/compute-plane/cluster-management/reference.md": return syncClusterManagementReference(content, catalog) default: return content, false, nil diff --git a/tools/docs-version-sync/main.go b/tools/docs-version-sync/main.go index 50869bf941..98773d47cd 100644 --- a/tools/docs-version-sync/main.go +++ b/tools/docs-version-sync/main.go @@ -42,7 +42,8 @@ func run(args []string) error { stackVersion := flags.String("stack-version", "", "self-managed stack version to fetch or inventory") computeStackVersion := flags.String("compute-stack-version", "", "compute-plane stack version to fetch") observabilityStackVersion := flags.String("observability-stack-version", "", "observability stack version to fetch") - qualificationVersion := flags.String("qualification-version", "", "documentation version for an exact QA-qualified three-stack release set (cp-X.Y.Z-compute-X.Y.Z-obs-X.Y.Z)") + freezeStack := flags.String("freeze-stack", "", "release_set stack whose documentation train is being frozen (control-plane, compute-plane, or observability)") + freezeTrain := flags.String("freeze-train", "", "documentation train to freeze for --freeze-stack ("+documentationTrainFormat+")") inventoryOutput := flags.String("generate-stack-inventory", "", "write a resolved stack inventory to this path") inventoryConfig := flags.String("inventory-config", "", "release inventory config path; defaults to the stack checkout") allowUnavailableSourceCharts := flags.Bool("allow-unavailable-source-charts", false, "use published charts when configured source paths are unavailable in a historical tag") @@ -69,7 +70,7 @@ func run(args []string) error { if *compareFrom == "" || *compareTo == "" { return fmt.Errorf("--compare-release-set-from and --compare-release-set-to must be used together") } - if *updateCatalog || *check || *inventoryOutput != "" || *qualificationVersion != "" || + if *updateCatalog || *check || *inventoryOutput != "" || *freezeStack != "" || *freezeTrain != "" || *stackVersion != "" || *computeStackVersion != "" || *observabilityStackVersion != "" || *inventoryConfig != "" || *allowUnavailableSourceCharts || *stackSourceTag != "" || *stackSourceCommit != "" { return fmt.Errorf("release-set comparison cannot be combined with catalog update, check, or inventory generation flags") @@ -111,17 +112,6 @@ func run(args []string) error { if !*updateCatalog && (*computeStackVersion != "" || *observabilityStackVersion != "") { return fmt.Errorf("--compute-stack-version and --observability-stack-version require --update-catalog") } - if *qualificationVersion != "" { - if !*updateCatalog { - return fmt.Errorf("--qualification-version requires --update-catalog") - } - if _, _, _, ok := parseReleaseSetDocumentationVersion(*qualificationVersion); !ok { - return fmt.Errorf("--qualification-version must use %s", releaseSetVersionFormat) - } - if *stackVersion == "" || *computeStackVersion == "" || *observabilityStackVersion == "" { - return fmt.Errorf("--qualification-version requires exact control-plane, compute-plane, and observability stack versions") - } - } if *stackSourceTag != "" || *stackSourceCommit != "" || *inventoryConfig != "" || *allowUnavailableSourceCharts { return fmt.Errorf("--stack-source-tag, --stack-source-commit, --inventory-config, and --allow-unavailable-source-charts require --generate-stack-inventory") } @@ -131,6 +121,20 @@ func run(args []string) error { if !filepath.IsAbs(*catalogPath) { *catalogPath = filepath.Join(repoRoot, *catalogPath) } + if *freezeStack != "" || *freezeTrain != "" { + if *freezeStack == "" || *freezeTrain == "" { + return fmt.Errorf("--freeze-stack and --freeze-train must be used together") + } + if *updateCatalog || *check || *stackVersion != "" || *computeStackVersion != "" || *observabilityStackVersion != "" { + return fmt.Errorf("--freeze-stack cannot be combined with catalog update, check, or stack version flags") + } + snapshotPath, err := freezeStackDocumentation(repoRoot, *catalogPath, *freezeStack, *freezeTrain) + if err != nil { + return err + } + fmt.Fprintf(os.Stderr, "wrote %s for %s train %s\n", relOrAbs(repoRoot, snapshotPath), *freezeStack, *freezeTrain) + return nil + } var catalog *Catalog if *updateCatalog { @@ -142,7 +146,7 @@ func run(args []string) error { selfManagedStackKey: *stackVersion, computePlaneStackKey: *computeStackVersion, observabilityStackKey: *observabilityStackVersion, - }, *qualificationVersion, base) + }, base) if err != nil { return err } diff --git a/tools/docs-version-sync/main_test.go b/tools/docs-version-sync/main_test.go index a8ecc023ea..367a9ca395 100644 --- a/tools/docs-version-sync/main_test.go +++ b/tools/docs-version-sync/main_test.go @@ -71,7 +71,7 @@ func TestRenderManifestHandlesNewNVCAAndNVCTImageAndHelmArtifacts(t *testing.T) t.Fatalf("Render failed: %v", err) } - computeServices := sectionBetween(t, got, "### Compute plane services and images", "### EA-only CVE-impacted artifacts") + computeServices := sectionBetween(t, got, "### Compute plane services and images", "### Observability Helm charts") for _, want := range []string{ "| `nvca` | `3.0.0-rc.13` | `compute-plane` | Required |", "| `nvca-operator` | `3.0.0-rc.13` | `compute-plane` | Required |", @@ -180,6 +180,24 @@ func TestCatalogRefreshPreservesVersionQualifiedPublications(t *testing.T) { } } +func TestCatalogRefreshPreservesCompatibility(t *testing.T) { + base := testCatalog() + base.Compatibility = []CompatibilityEntry{ + {Stack: "control-plane", Train: "1.0", CompatibleWith: map[string]string{"compute-plane": "1.0+", "observability": "1.0+"}}, + } + + updated := refreshCatalogFromArtifacts("1.0.0", []Artifact{{ + Name: "nvcf-grpc-proxy", + Type: ArtifactTypeImage, + Registry: defaultImageRegistry, + Version: "1.30.0", + }}, base) + + if len(updated.Compatibility) != 1 || updated.Compatibility[0].CompatibleWith["compute-plane"] != "1.0+" { + t.Fatalf("compatibility = %#v, want the base entries preserved across a refresh", updated.Compatibility) + } +} + func TestCatalogConstructionWithoutBaseRendersUnverifiedArtifactsAsPending(t *testing.T) { catalog := refreshCatalogFromArtifacts("0.9.1", []Artifact{ {Name: "helm-nvca-operator", Type: ArtifactTypeChart, Registry: defaultChartRegistry, Version: "1.21.3"}, @@ -637,7 +655,7 @@ func TestSyncInlineSelfManagedNVCAOperatorVersionTable(t *testing.T) { content := "| Chart | `helm-nvca-operator` |\n| --- | --- |\n| Version | `1.6.7` |\n\n" + "The compute-plane Helmfile installs the operator.\n" - got, changed, err := SyncInlineVersions("docs/user/cluster-management/self-managed.md", content, catalog) + got, changed, err := SyncInlineVersions("docs/compute-plane/cluster-management/self-managed.md", content, catalog) if err != nil { t.Fatalf("SyncInlineVersions failed: %v", err) } @@ -662,7 +680,7 @@ func TestSyncInlineImageMirroringNVCAOperatorChartVersions(t *testing.T) { "# This creates: nvca-operator-1.2.9.tgz\n" + "helm push nvca-operator-1.2.9.tgz oci://example.test/repo\n" - got, changed, err := SyncInlineVersions("docs/user/image-mirroring.md", content, catalog) + got, changed, err := SyncInlineVersions("docs/overview/image-mirroring.md", content, catalog) if err != nil { t.Fatalf("SyncInlineVersions failed: %v", err) } @@ -706,7 +724,7 @@ func TestSyncInlineImageMirroringUsesTraditionalPublicHelmChart(t *testing.T) { "# This creates: helm-nvca-operator-1.12.6.tgz\n" + "helm push helm-nvca-operator-1.12.6.tgz oci://example.test/repo\n" - got, changed, err := SyncInlineVersions("docs/user/image-mirroring.md", content, catalog) + got, changed, err := SyncInlineVersions("docs/overview/image-mirroring.md", content, catalog) if err != nil { t.Fatalf("SyncInlineVersions failed: %v", err) } @@ -740,7 +758,7 @@ func TestSyncInlineImageMirroringUsesTraditionalUpstreamHelmChart(t *testing.T) "# This creates: helm-nvca-operator-1.12.6.tgz\n" + "helm push helm-nvca-operator-1.12.6.tgz oci://example.test/repo\n" - got, changed, err := SyncInlineVersions("docs/user/image-mirroring.md", content, catalog) + got, changed, err := SyncInlineVersions("docs/overview/image-mirroring.md", content, catalog) if err != nil { t.Fatalf("SyncInlineVersions failed: %v", err) } @@ -752,7 +770,7 @@ func TestSyncInlineImageMirroringUsesTraditionalUpstreamHelmChart(t *testing.T) t.Fatalf("updated content missing %q:\n%s", want, got) } - gotAgain, changedAgain, err := SyncInlineVersions("docs/user/image-mirroring.md", got, catalog) + gotAgain, changedAgain, err := SyncInlineVersions("docs/overview/image-mirroring.md", got, catalog) if err != nil { t.Fatalf("second SyncInlineVersions failed: %v", err) } @@ -763,7 +781,7 @@ func TestSyncInlineImageMirroringUsesTraditionalUpstreamHelmChart(t *testing.T) func TestSyncDocsCheckModeDetectsDiff(t *testing.T) { tmp := t.TempDir() - writeFile(t, filepath.Join(tmp, "docs/user/manifest.md"), `before + writeFile(t, filepath.Join(tmp, "docs/overview/manifest.md"), `before {/* docs-version-sync:BEGIN manifest-artifact-registry-paths */} stale {/* docs-version-sync:END manifest-artifact-registry-paths */} @@ -772,7 +790,7 @@ after catalog := testCatalog() catalog.Outputs = []OutputFile{{ - Path: "docs/user/manifest.md", + Path: "docs/overview/manifest.md", Blocks: []OutputBlock{{ Marker: "manifest-artifact-registry-paths", Renderer: "manifest-artifact-registry-paths", @@ -784,7 +802,7 @@ after t.Fatalf("SyncDocs error = %v, want ErrCheckFailed", err) } - got, err := os.ReadFile(filepath.Join(tmp, "docs/user/manifest.md")) + got, err := os.ReadFile(filepath.Join(tmp, "docs/overview/manifest.md")) if err != nil { t.Fatalf("read manifest: %v", err) } @@ -871,11 +889,11 @@ func TestFindRepoRootFromRequiresBothPublicCheckoutSentinels(t *testing.T) { func TestSyncDocsRejectsMissingMarker(t *testing.T) { tmp := t.TempDir() - writeFile(t, filepath.Join(tmp, "docs/user/manifest.md"), "no generated marker\n") + writeFile(t, filepath.Join(tmp, "docs/overview/manifest.md"), "no generated marker\n") catalog := testCatalog() catalog.Outputs = []OutputFile{{ - Path: "docs/user/manifest.md", + Path: "docs/overview/manifest.md", Blocks: []OutputBlock{{ Marker: "manifest-artifact-registry-paths", Renderer: "manifest-artifact-registry-paths", @@ -924,8 +942,32 @@ func TestValidateCatalogRejectsOutputOutsideDocsUser(t *testing.T) { if err == nil { t.Fatal("ValidateCatalog succeeded, want path rejection") } - if !strings.Contains(err.Error(), "outside docs/user") { - t.Fatalf("error = %q, want outside docs/user", err) + if !strings.Contains(err.Error(), "outside the documentation product trees") { + t.Fatalf("error = %q, want outside product trees", err) + } +} + +func TestValidateCatalogAcceptsEveryProductTreeOutput(t *testing.T) { + for _, path := range []string{ + "docs/overview/compatibility-matrix.md", + "docs/self-managed/installation.md", + "docs/compute-plane/cluster-management/reference.md", + "docs/observability/observability.md", + } { + catalog := testCatalog() + catalog.Outputs = []OutputFile{{Path: path}} + if err := ValidateCatalog(catalog); err != nil { + t.Fatalf("ValidateCatalog rejected %s: %v", path, err) + } + } +} + +func TestValidateCatalogRejectsFrozenProductTreeOutput(t *testing.T) { + catalog := testCatalog() + catalog.Outputs = []OutputFile{{Path: "docs/observability-1.3/observability.md"}} + err := ValidateCatalog(catalog) + if err == nil || !strings.Contains(err.Error(), "frozen docs are generated from the product trees") { + t.Fatalf("ValidateCatalog error = %v, want frozen tree rejection", err) } } @@ -938,23 +980,221 @@ func TestValidateTargetRejectsNonMainTargets(t *testing.T) { } } -func TestRunRejectsInvalidReleaseSetQualificationVersion(t *testing.T) { - err := run([]string{"--update-catalog", "--qualification-version", "1.2.3"}) - if err == nil || !strings.Contains(err.Error(), "must use "+releaseSetVersionFormat) { - t.Fatalf("run error = %v, want release-set version format rejection", err) +func TestRunRejectsFreezeFlagsWithoutEachOther(t *testing.T) { + err := run([]string{"--freeze-stack", "observability"}) + if err == nil || !strings.Contains(err.Error(), "must be used together") { + t.Fatalf("run error = %v, want paired flag rejection", err) + } + err = run([]string{"--freeze-stack", "observability", "--freeze-train", "1.1", "--update-catalog"}) + if err == nil || !strings.Contains(err.Error(), "cannot be combined") { + t.Fatalf("run error = %v, want combination rejection", err) + } +} + +func TestFreezeStackDocumentationWritesQualifiedSnapshotForOneStack(t *testing.T) { + tmp := t.TempDir() + catalogPath := filepath.Join(tmp, "docs", "version-catalog", "main.yaml") + catalog := testCatalogWithReleaseSet() + if err := WriteCatalog(catalogPath, catalog); err != nil { + t.Fatal(err) + } + + snapshotPath, err := freezeStackDocumentation(tmp, catalogPath, "observability", "3.4") + if err != nil { + t.Fatalf("freezeStackDocumentation failed: %v", err) + } + if filepath.Base(snapshotPath) != "observability-3.4.yaml" { + t.Fatalf("snapshot path = %s, want observability-3.4.yaml", snapshotPath) + } + snapshot, err := LoadCatalog(snapshotPath) + if err != nil { + t.Fatal(err) + } + if snapshot.ReleaseSet.Stacks.Observability.Status != ReleaseSetQualified || snapshot.ReleaseSet.Stacks.Observability.DocumentationVersion != "3.4" { + t.Fatalf("snapshot observability = %#v, want qualified 3.4", snapshot.ReleaseSet.Stacks.Observability) + } + if snapshot.ReleaseSet.Stacks.ControlPlane.Status != ReleaseSetDevelopment || snapshot.ReleaseSet.Stacks.ComputePlane.Status != ReleaseSetDevelopment { + t.Fatalf("snapshot froze other stacks: %#v", snapshot.ReleaseSet.Stacks) + } + main, err := LoadCatalog(catalogPath) + if err != nil { + t.Fatal(err) + } + if main.ReleaseSet.Stacks.Observability.Status != ReleaseSetDevelopment { + t.Fatalf("main catalog was modified by freeze: %#v", main.ReleaseSet.Stacks.Observability) + } + + if _, err := freezeStackDocumentation(tmp, catalogPath, "observability", "3.4"); err == nil || !strings.Contains(err.Error(), "refusing to overwrite") { + t.Fatalf("second freeze error = %v, want overwrite refusal", err) + } +} + +func TestFreezeStackDocumentationRejectsTrainMismatch(t *testing.T) { + tmp := t.TempDir() + catalogPath := filepath.Join(tmp, "docs", "version-catalog", "main.yaml") + catalog := testCatalogWithReleaseSet() + if err := WriteCatalog(catalogPath, catalog); err != nil { + t.Fatal(err) + } + + if _, err := freezeStackDocumentation(tmp, catalogPath, "compute-plane", "2.4"); err == nil || !strings.Contains(err.Error(), "belongs to train 2.3, not 2.4") { + t.Fatalf("freeze error = %v, want train mismatch", err) + } + if _, err := freezeStackDocumentation(tmp, catalogPath, "control-plane", "1.2.3"); err == nil || !strings.Contains(err.Error(), "must use "+documentationTrainFormat) { + t.Fatalf("freeze error = %v, want train format rejection", err) + } + if _, err := freezeStackDocumentation(tmp, catalogPath, "self-managed", "1.2"); err == nil || !strings.Contains(err.Error(), "unknown release_set stack") { + t.Fatalf("freeze error = %v, want unknown stack rejection", err) + } + +} + +func TestFreezeStackDocumentationAllowsPendingPublications(t *testing.T) { + tmp := t.TempDir() + catalogPath := filepath.Join(tmp, "docs", "version-catalog", "main.yaml") + catalog := testCatalogWithReleaseSet() + catalog.PublicationPending = []string{"llm-api-gateway"} + if err := WriteCatalog(catalogPath, catalog); err != nil { + t.Fatal(err) + } + snapshot, err := freezeStackDocumentation(tmp, catalogPath, "control-plane", "1.2") + if err != nil { + t.Fatalf("freeze with pending publications failed: %v", err) + } + frozen, err := LoadCatalog(snapshot) + if err != nil { + t.Fatal(err) + } + if len(frozen.PublicationPending) != 1 || frozen.PublicationPending[0] != "llm-api-gateway" { + t.Fatalf("frozen snapshot lost publication_pending: %#v", frozen.PublicationPending) } } func TestValidateCatalogRejectsPartiallyPopulatedReleaseSet(t *testing.T) { - catalog := testCatalog() - catalog.ReleaseSet.Status = ReleaseSetDevelopment + catalog := testCatalogWithReleaseSet() + catalog.ReleaseSet.Stacks.ComputePlane.DocumentationVersion = "" err := ValidateCatalog(catalog) - if err == nil || !strings.Contains(err.Error(), "documentation_version must be non-empty") { + if err == nil || !strings.Contains(err.Error(), "compute-plane documentation_version must be non-empty") { t.Fatalf("ValidateCatalog error = %v, want partial release_set rejection", err) } } +func TestValidateCatalogRejectsMalformedCompatibility(t *testing.T) { + for name, entries := range map[string][]CompatibilityEntry{ + "unknown stack": {{Stack: "self-managed", Train: "1.1", CompatibleWith: map[string]string{"compute-plane": "1.1+", "observability": "1.1+"}}}, + "full version": {{Stack: "control-plane", Train: "1.1.0", CompatibleWith: map[string]string{"compute-plane": "1.1+", "observability": "1.1+"}}}, + "own stack": {{Stack: "control-plane", Train: "1.1", CompatibleWith: map[string]string{"control-plane": "1.1+", "observability": "1.1+"}}}, + "missing stack": {{Stack: "control-plane", Train: "1.1", CompatibleWith: map[string]string{"compute-plane": "1.1+"}}}, + "empty requirement": {{Stack: "control-plane", Train: "1.1", CompatibleWith: map[string]string{"compute-plane": "", "observability": "1.1+"}}}, + "patch requirement": {{Stack: "control-plane", Train: "1.1", CompatibleWith: map[string]string{"compute-plane": "1.1.0+", "observability": "1.1+"}}}, + "range requirement": {{Stack: "control-plane", Train: "1.1", CompatibleWith: map[string]string{"compute-plane": ">=1.1", "observability": "1.1+"}}}, + "duplicate": { + {Stack: "control-plane", Train: "1.1", CompatibleWith: map[string]string{"compute-plane": "1.1+", "observability": "1.1+"}}, + {Stack: "control-plane", Train: "1.1", CompatibleWith: map[string]string{"compute-plane": "1.0+", "observability": "1.0+"}}, + }, + } { + catalog := testCatalog() + catalog.Compatibility = entries + if err := ValidateCatalog(catalog); err == nil { + t.Fatalf("ValidateCatalog accepted %s compatibility", name) + } + } +} + +func TestPreserveQualifiedDocumentationAcrossRefresh(t *testing.T) { + base := testCatalogWithReleaseSet().ReleaseSet + base.Stacks.ControlPlane.DocumentationVersion = "1.2" + base.Stacks.ControlPlane.Status = ReleaseSetQualified + base.Stacks.Observability.DocumentationVersion = "3.4" + base.Stacks.Observability.Status = ReleaseSetQualified + + refreshed := testCatalogWithReleaseSet().ReleaseSet + refreshed.Stacks.ControlPlane.Version = "1.2.9" + refreshed.Stacks.Observability.Version = "3.5.0" + + if err := preserveQualifiedDocumentation(&refreshed, base); err != nil { + t.Fatalf("preserveQualifiedDocumentation failed: %v", err) + } + if refreshed.Stacks.ControlPlane.Status != ReleaseSetQualified || refreshed.Stacks.ControlPlane.DocumentationVersion != "1.2" { + t.Fatalf("control plane patch release lost its qualified train: %#v", refreshed.Stacks.ControlPlane) + } + if refreshed.Stacks.Observability.Status != ReleaseSetDevelopment || refreshed.Stacks.Observability.DocumentationVersion != "dev" { + t.Fatalf("observability release on a new train should fall back to dev: %#v", refreshed.Stacks.Observability) + } + if refreshed.Stacks.ComputePlane.Status != ReleaseSetDevelopment { + t.Fatalf("compute plane was never qualified and should stay dev: %#v", refreshed.Stacks.ComputePlane) + } + + if err := preserveQualifiedDocumentation(&refreshed, ReleaseSetMetadata{}); err != nil { + t.Fatalf("empty base should be a no-op, got %v", err) + } +} + +func TestRenderCompatibilityMatrix(t *testing.T) { + catalog := testCatalogWithReleaseSet() + catalog.ReleaseSet.Stacks.Observability.DocumentationVersion = "3.4" + catalog.ReleaseSet.Stacks.Observability.Status = ReleaseSetQualified + catalog.Compatibility = []CompatibilityEntry{ + {Stack: "observability", Train: "3.3", CompatibleWith: map[string]string{"control-plane": "1.2", "compute-plane": "2.3+"}}, + {Stack: "observability", Train: "3.4", CompatibleWith: map[string]string{"control-plane": "1.1+", "compute-plane": "2.3+"}}, + {Stack: "control-plane", Train: "1.2", CompatibleWith: map[string]string{"compute-plane": "2.3+", "observability": "3.3+"}}, + } + got, err := Render("compatibility-matrix", catalog) + if err != nil { + t.Fatalf("Render failed: %v", err) + } + for _, want := range []string{ + "## Current stack releases", + "| Stack | Latest release | Source tag |", + "| [Self-managed (control plane)](/nvcf/self-managed/) | `1.2.3` | `deploy/stacks/self-managed/v1.2.3` |", + "| [Compute plane](/nvcf/compute-plane/) | `2.3.4` | `deploy/stacks/nvcf-compute-plane/v2.3.4` |", + "| [Observability](/nvcf/observability/) | `3.4.5` | `deploy/stacks/observability/v3.4.5` |", + "## Compatible stack versions", + "| Stack | Release | Works with |", + "| Self-managed (control plane) | `1.2` | Compute plane `2.3` or later, Observability `3.3` or later |", + "| Observability | `3.4` | Self-managed (control plane) `1.1` or later, Compute plane `2.3` or later |", + "| Observability | `3.3` | Self-managed (control plane) `1.2` only, Compute plane `2.3` or later |", + } { + if !strings.Contains(got, want) { + t.Fatalf("rendered matrix missing %q:\n%s", want, got) + } + } + if strings.Index(got, "| Self-managed (control plane) | `1.2` |") > strings.Index(got, "| Observability | `3.4` |") { + t.Fatalf("rows are not ordered by stack:\n%s", got) + } + if strings.Index(got, "| Observability | `3.4` |") > strings.Index(got, "| Observability | `3.3` |") { + t.Fatalf("trains are not ordered newest first:\n%s", got) + } + + catalog.Compatibility = nil + if _, err := Render("compatibility-matrix", catalog); err == nil { + t.Fatal("Render succeeded without compatibility entries") + } +} + +func testCatalogWithReleaseSet() *Catalog { + catalog := testCatalog() + catalog.ReleaseSet = ReleaseSetMetadata{Stacks: ReleaseSetStacks{ + ControlPlane: StackReleaseMetadata{ + Version: "1.2.3", SourceTag: stackInventorySpecs[0].TagPrefix + "1.2.3", + SourceCommit: strings.Repeat("a", 40), InventoryAsset: stackInventorySpecs[0].AssetName, + DocumentationVersion: "dev", Status: ReleaseSetDevelopment, + }, + ComputePlane: StackReleaseMetadata{ + Version: "2.3.4", SourceTag: stackInventorySpecs[1].TagPrefix + "2.3.4", + SourceCommit: strings.Repeat("b", 40), InventoryAsset: stackInventorySpecs[1].AssetName, + DocumentationVersion: "dev", Status: ReleaseSetDevelopment, + }, + Observability: StackReleaseMetadata{ + Version: "3.4.5", SourceTag: stackInventorySpecs[2].TagPrefix + "3.4.5", + SourceCommit: strings.Repeat("c", 40), InventoryAsset: stackInventorySpecs[2].AssetName, + DocumentationVersion: "dev", Status: ReleaseSetDevelopment, + }, + }} + return catalog +} + func testCatalog() *Catalog { return &Catalog{ Version: 1, diff --git a/tools/docs-version-sync/manifest.go b/tools/docs-version-sync/manifest.go index 6ad3f6b327..6a0cd1d819 100644 --- a/tools/docs-version-sync/manifest.go +++ b/tools/docs-version-sync/manifest.go @@ -179,29 +179,33 @@ func renderManifestArtifactRegistryPaths(catalog *Catalog) (string, error) { } func renderReleaseSetSummary(releaseSet ReleaseSetMetadata) string { - if releaseSet.DocumentationVersion == "" { + if releaseSet == (ReleaseSetMetadata{}) { return "" } - return fmt.Sprintf("### Stack release set\n\nDocumentation: `%s` (%s)\n\n| Stack | Version | Source tag |\n| --- | --- | --- |\n| Control plane | `%s` | `%s` |\n| Compute plane | `%s` | `%s` |\n| Observability | `%s` | `%s` |\n\n", - releaseSet.DocumentationVersion, - releaseSet.Status, - releaseSet.Stacks.ControlPlane.Version, - releaseSet.Stacks.ControlPlane.SourceTag, - releaseSet.Stacks.ComputePlane.Version, - releaseSet.Stacks.ComputePlane.SourceTag, - releaseSet.Stacks.Observability.Version, - releaseSet.Stacks.Observability.SourceTag, - ) + var b strings.Builder + b.WriteString("### Stack releases\n\n") + b.WriteString("| Stack | Version | Source tag |\n| --- | --- | --- |\n") + for _, stack := range releaseSetStackNames { + metadata, _ := releaseSet.Stacks.byName(stack) + slug, _ := documentationProductSlug(stack) + b.WriteString(fmt.Sprintf("| [%s](/nvcf/%s/) | `%s` | `%s` |\n", + documentationStackDisplayName(stack), slug, metadata.Version, metadata.SourceTag)) + } + b.WriteString("\n") + return b.String() } func renderManifestTables(entries []resolvedManifestEntry) string { var b strings.Builder for _, section := range manifestSections { + sectionEntries := manifestEntriesForSection(entries, section) + if section.Kind == ManifestKindEACVE && len(sectionEntries) == 0 { + continue + } b.WriteString("### " + section.Heading + "\n\n") if section.Description != "" { b.WriteString(section.Description + "\n\n") } - sectionEntries := manifestEntriesForSection(entries, section) if section.Kind == ManifestKindResource { b.WriteString("| Artifact | Version | Stack | Description | Distribution | Source code |\n") b.WriteString("| --- | --- | --- | --- | --- | --- |\n") diff --git a/tools/docs-version-sync/manifest_metadata_test.go b/tools/docs-version-sync/manifest_metadata_test.go index 40e95578b9..eba653d31a 100644 --- a/tools/docs-version-sync/manifest_metadata_test.go +++ b/tools/docs-version-sync/manifest_metadata_test.go @@ -247,9 +247,7 @@ func TestManifestMetadataClassifiesAllArtifacts(t *testing.T) { t.Fatalf("manifest classification mismatch: missing=%v extra=%v", missing, extra) } - sort.Strings(eaCVE) - wantEACVE := []string{"nvcf-cassandra-migrations"} - if strings.Join(eaCVE, ",") != strings.Join(wantEACVE, ",") { - t.Fatalf("EA-CVE entries = %v, want %v", eaCVE, wantEACVE) + if len(eaCVE) != 0 { + t.Fatalf("EA-CVE entries = %v, want none: the latest release has no CVE-impacted EA artifacts", eaCVE) } } diff --git a/tools/docs-version-sync/manifest_test.go b/tools/docs-version-sync/manifest_test.go index b375bdd604..6ae00b096a 100644 --- a/tools/docs-version-sync/manifest_test.go +++ b/tools/docs-version-sync/manifest_test.go @@ -55,24 +55,28 @@ func TestResolveManifestEntriesRejectsUnclassifiedArtifact(t *testing.T) { } } -func TestResolveManifestEntriesKeepsEACVEArtifactsOutOfServiceSection(t *testing.T) { +func TestResolveManifestEntriesListsCassandraMigrationsAsServiceImage(t *testing.T) { entries, err := resolveManifestEntries(loadMainCatalog(t)) if err != nil { t.Fatalf("resolveManifestEntries failed: %v", err) } var eaIDs []string + foundMigrations := false for _, entry := range entries { if entry.Kind == ManifestKindEACVE { eaIDs = append(eaIDs, entry.ID) } if entry.Kind == ManifestKindServiceImage && entry.ID == "nvcf-cassandra-migrations" { - t.Fatalf("EA artifact %s also appears as a service image", entry.ID) + foundMigrations = true } } sort.Strings(eaIDs) - if got, want := strings.Join(eaIDs, ","), "nvcf-cassandra-migrations"; got != want { - t.Fatalf("EA entries = %q, want %q", got, want) + if len(eaIDs) != 0 { + t.Fatalf("EA entries = %v, want none", eaIDs) + } + if !foundMigrations { + t.Fatal("nvcf-cassandra-migrations must be listed as a control plane service image") } } @@ -87,9 +91,11 @@ func TestRenderManifestTable(t *testing.T) { "### Control plane services and images", "### Compute plane Helm charts", "### Compute plane services and images", - "### EA-only CVE-impacted artifacts", "### Tools and deployment resources", } + if strings.Contains(got, "### EA-only CVE-impacted artifacts") { + t.Fatalf("rendered manifest shows an EA-only CVE section with no entries:\n%s", got) + } last := -1 for _, want := range wantInOrder { index := strings.Index(got, want) @@ -104,7 +110,6 @@ func TestRenderManifestTable(t *testing.T) { for _, want := range []string{ "| Artifact | Version | Stack | Required | Description | Distribution | Source code |", - "These Early Access artifacts have known CVE impact.", "[GitHub](https://github.com/NVIDIA/nvcf/tree/main/deploy/helm/nats)", "[Upstream](https://github.com/nats-io/k8s)", "`nvcf-cassandra-migrations`", diff --git a/tools/docs-version-sync/render.go b/tools/docs-version-sync/render.go index 7948fd98e4..cf3e74b9f8 100644 --- a/tools/docs-version-sync/render.go +++ b/tools/docs-version-sync/render.go @@ -42,6 +42,8 @@ func Render(renderer string, catalog *Catalog) (string, error) { return renderImageMirroringObservabilityStackSnippet(catalog) case "image-mirroring-cli-snippet": return renderImageMirroringCLISnippet(catalog) + case "compatibility-matrix": + return renderCompatibilityMatrix(catalog) default: return "", fmt.Errorf("unknown renderer %q", renderer) } diff --git a/tools/docs-version-sync/stack_consistency_test.go b/tools/docs-version-sync/stack_consistency_test.go index 0a67954282..5c7a7f4ab2 100644 --- a/tools/docs-version-sync/stack_consistency_test.go +++ b/tools/docs-version-sync/stack_consistency_test.go @@ -205,7 +205,7 @@ func TestPendingPublicationsNeverRenderPrivateRegistryPaths(t *testing.T) { if err != nil { t.Fatal(err) } - imageMirroringPath := filepath.Join(root, "docs", "user", "image-mirroring.md") + imageMirroringPath := filepath.Join(root, "docs", "overview", "image-mirroring.md") imageMirroring, err := os.ReadFile(imageMirroringPath) if err != nil { t.Fatal(err) diff --git a/tools/docs-version-sync/stack_inventory.go b/tools/docs-version-sync/stack_inventory.go index 7c1b87456f..85aa7aad2c 100644 --- a/tools/docs-version-sync/stack_inventory.go +++ b/tools/docs-version-sync/stack_inventory.go @@ -77,7 +77,38 @@ func stackKeyForPlane(plane string) (string, error) { } } -func releaseSetFromInventories(inventories map[string]resolvedStackInventory, documentationVersion string, status ReleaseSetStatus) (ReleaseSetMetadata, error) { +// preserveQualifiedDocumentation keeps a stack's frozen documentation train +// across a refresh while the released version stays on that train. A release +// on a new train falls back to development documentation until it is frozen. +func preserveQualifiedDocumentation(refreshed *ReleaseSetMetadata, base ReleaseSetMetadata) error { + if base == (ReleaseSetMetadata{}) { + return nil + } + for _, stack := range releaseSetStackNames { + previous, err := base.Stacks.byName(stack) + if err != nil { + return err + } + if previous.Status != ReleaseSetQualified { + continue + } + current, err := refreshed.Stacks.byName(stack) + if err != nil { + return err + } + train, ok := releaseTrain(current.Version) + if !ok || train != previous.DocumentationVersion { + continue + } + current.DocumentationVersion = previous.DocumentationVersion + current.Status = ReleaseSetQualified + } + return nil +} + +// releaseSetFromInventories records the current release of each stack as +// development documentation. Freezing a stack train marks that stack qualified. +func releaseSetFromInventories(inventories map[string]resolvedStackInventory) (ReleaseSetMetadata, error) { metadata := func(key string) (StackReleaseMetadata, error) { spec, err := stackInventorySpecByKey(key) if err != nil { @@ -88,10 +119,12 @@ func releaseSetFromInventories(inventories map[string]resolvedStackInventory, do return StackReleaseMetadata{}, fmt.Errorf("%s inventory is required for release set metadata", key) } return StackReleaseMetadata{ - Version: inventory.Source.Version, - SourceTag: inventory.Source.Tag, - SourceCommit: inventory.Source.Commit, - InventoryAsset: spec.AssetName, + Version: inventory.Source.Version, + SourceTag: inventory.Source.Tag, + SourceCommit: inventory.Source.Commit, + InventoryAsset: spec.AssetName, + DocumentationVersion: "dev", + Status: ReleaseSetDevelopment, }, nil } controlPlane, err := metadata(selfManagedStackKey) @@ -107,8 +140,6 @@ func releaseSetFromInventories(inventories map[string]resolvedStackInventory, do return ReleaseSetMetadata{}, err } return ReleaseSetMetadata{ - DocumentationVersion: documentationVersion, - Status: status, Stacks: ReleaseSetStacks{ ControlPlane: controlPlane, ComputePlane: computePlane, @@ -116,7 +147,3 @@ func releaseSetFromInventories(inventories map[string]resolvedStackInventory, do }, }, nil } - -func (releaseSet ReleaseSetMetadata) sameStackReleases(other ReleaseSetMetadata) bool { - return releaseSet.Stacks == other.Stacks -} diff --git a/tools/docs-version-sync/stack_inventory_test.go b/tools/docs-version-sync/stack_inventory_test.go index dc34608306..e2f2815a9d 100644 --- a/tools/docs-version-sync/stack_inventory_test.go +++ b/tools/docs-version-sync/stack_inventory_test.go @@ -41,7 +41,7 @@ func TestReleaseSetRecordsAllThreeImmutableSources(t *testing.T) { Commit: strings.Repeat(string(rune('a'+index)), 40), }} } - releaseSet, err := releaseSetFromInventories(inventories, "cp-1.0.0-compute-1.1.0-obs-1.2.0", ReleaseSetQualified) + releaseSet, err := releaseSetFromInventories(inventories) if err != nil { t.Fatal(err) } @@ -53,36 +53,55 @@ func TestReleaseSetRecordsAllThreeImmutableSources(t *testing.T) { releaseSet.Stacks.Observability.InventoryAsset != stackInventorySpecs[2].AssetName { t.Fatalf("release set assets = %#v", releaseSet.Stacks) } - - releaseSet.DocumentationVersion = "cp-1.0.1-compute-1.1.0-obs-1.2.0" - if err := validateReleaseSet(releaseSet); err == nil || !strings.Contains(err.Error(), "must be cp-1.0.0-compute-1.1.0-obs-1.2.0") { - t.Fatalf("validateReleaseSet error = %v, want stack version mismatch", err) + if releaseSet.Stacks.Observability.DocumentationVersion != "dev" || releaseSet.Stacks.Observability.Status != ReleaseSetDevelopment { + t.Fatalf("release set observability = %#v, want development docs", releaseSet.Stacks.Observability) } } -func TestReleaseSetStackComparisonIncludesImmutableSourceMetadata(t *testing.T) { +func TestValidateReleaseSetChecksPerStackDocumentationTrain(t *testing.T) { releaseSet := ReleaseSetMetadata{Stacks: ReleaseSetStacks{ ControlPlane: StackReleaseMetadata{ Version: "1.2.3", SourceTag: stackInventorySpecs[0].TagPrefix + "1.2.3", SourceCommit: strings.Repeat("a", 40), InventoryAsset: stackInventorySpecs[0].AssetName, + DocumentationVersion: "dev", Status: ReleaseSetDevelopment, }, ComputePlane: StackReleaseMetadata{ Version: "2.3.4", SourceTag: stackInventorySpecs[1].TagPrefix + "2.3.4", SourceCommit: strings.Repeat("b", 40), InventoryAsset: stackInventorySpecs[1].AssetName, + DocumentationVersion: "dev", Status: ReleaseSetDevelopment, }, Observability: StackReleaseMetadata{ Version: "3.4.5", SourceTag: stackInventorySpecs[2].TagPrefix + "3.4.5", SourceCommit: strings.Repeat("c", 40), InventoryAsset: stackInventorySpecs[2].AssetName, + DocumentationVersion: "3.4", Status: ReleaseSetQualified, }, }} - if !releaseSet.sameStackReleases(releaseSet) { - t.Fatal("identical stack metadata did not match") + if err := validateReleaseSet(releaseSet); err != nil { + t.Fatalf("validateReleaseSet rejected a qualified observability train: %v", err) + } + + wrongTrain := releaseSet + wrongTrain.Stacks.Observability.DocumentationVersion = "3.5" + if err := validateReleaseSet(wrongTrain); err == nil || !strings.Contains(err.Error(), "must be 3.4 for version 3.4.5") { + t.Fatalf("validateReleaseSet error = %v, want train mismatch", err) + } + + fullVersion := releaseSet + fullVersion.Stacks.Observability.DocumentationVersion = "3.4.5" + if err := validateReleaseSet(fullVersion); err == nil || !strings.Contains(err.Error(), "must use "+documentationTrainFormat) { + t.Fatalf("validateReleaseSet error = %v, want train format rejection", err) + } + + devWithTrain := releaseSet + devWithTrain.Stacks.ControlPlane.DocumentationVersion = "1.2" + if err := validateReleaseSet(devWithTrain); err == nil || !strings.Contains(err.Error(), "documentation_version must be dev") { + t.Fatalf("validateReleaseSet error = %v, want development docs rejection", err) } - changed := releaseSet - changed.Stacks.Observability.SourceCommit = strings.Repeat("d", 40) - if releaseSet.sameStackReleases(changed) { - t.Fatal("release sets with different immutable source commits matched") + missing := releaseSet + missing.Stacks.ComputePlane.DocumentationVersion = "" + if err := validateReleaseSet(missing); err == nil || !strings.Contains(err.Error(), "documentation_version must be non-empty") { + t.Fatalf("validateReleaseSet error = %v, want missing documentation version rejection", err) } } diff --git a/tools/ncp-local-cluster/docs/dynamo-operator.md b/tools/ncp-local-cluster/docs/dynamo-operator.md index 7ca7cfebd2..c5810e7b97 100644 --- a/tools/ncp-local-cluster/docs/dynamo-operator.md +++ b/tools/ncp-local-cluster/docs/dynamo-operator.md @@ -22,7 +22,7 @@ NVCF API; both are cluster infrastructure that the Dynamo Operator relies on. ## 1. Install KAI Scheduler -Follow the [KAI Scheduler installation guide](/docs/user/cluster-management/kai-scheduler.md). +Follow the [KAI Scheduler installation guide](../../../docs/compute-plane/cluster-management/kai-scheduler.md). **Note:** topology-aware scheduling with Dynamo requires Grove v0.14.0 or later, so ensure that version is set instead of the default version if you want to test that behavior. diff --git a/tools/ncp-local-cluster/tests/test-gateway-timeout-compatibility.sh b/tools/ncp-local-cluster/tests/test-gateway-timeout-compatibility.sh index c5ff6bd929..a8372eb643 100755 --- a/tools/ncp-local-cluster/tests/test-gateway-timeout-compatibility.sh +++ b/tools/ncp-local-cluster/tests/test-gateway-timeout-compatibility.sh @@ -70,7 +70,7 @@ export TEST_CALL_LOG="$call_log" export MOCK_CRD_MARKER="$test_dir/crd-seen" setup_script="$local_cluster_dir/scripts/setup-gateway-api.sh" -docs_file="$repo_dir/docs/user/gateway-routing.md" +docs_file="$repo_dir/docs/self-managed/gateway-routing.md" setup_version="$(sed -n 's/^ENVOY_GATEWAY_VERSION="\([^"]*\)"$/\1/p' "$setup_script")" docs_version="$( awk ' diff --git a/tools/scripts/cut-docs-version.sh b/tools/scripts/cut-docs-version.sh index 5ebfc1467e..23752e55b9 100755 --- a/tools/scripts/cut-docs-version.sh +++ b/tools/scripts/cut-docs-version.sh @@ -2,162 +2,81 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Snapshot the top-of-tree docs/user tree into a versioned subdirectory and -# generate a matching Fern navigation file. See the internal docs-versioning -# plan for the design. +# Freeze one stack's documentation train: snapshot docs// into +# docs/-/, generate the matching Fern product version file, and +# write the catalog snapshot. Overview documentation is unversioned and is +# never cut. Each stack is cut independently. set -euo pipefail -if [[ $# -ne 1 ]]; then - echo "Usage: $0 " >&2 - echo "Example: $0 cp-0.20.6-compute-0.4.4-obs-0.2.2" >&2 +usage() { + echo "Usage: $0 --stack --train " >&2 + echo "Example: $0 --stack observability --train 1.3" >&2 exit 1 -fi - -VERSION="$1" -DISPLAY="$VERSION" +} + +STACK="" +TRAIN="" +while [[ $# -gt 0 ]]; do + case "$1" in + --stack) STACK="${2:-}"; shift 2 ;; + --train) TRAIN="${2:-}"; shift 2 ;; + *) usage ;; + esac +done +[[ -n "$STACK" && -n "$TRAIN" ]] || usage + +case "$STACK" in + self-managed) RELEASE_SET_STACK="control-plane" ;; + compute-plane|observability) RELEASE_SET_STACK="$STACK" ;; + *) echo "Unknown stack $STACK; want self-managed, compute-plane, or observability" >&2; exit 1 ;; +esac +[[ "$TRAIN" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || { echo "Train must use X.Y, got $TRAIN" >&2; exit 1; } root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -docs_src="$root/docs/user" -docs_dst="$root/docs/$VERSION" -nav_src="$root/fern/versions/dev.yml" -nav_dst="$root/fern/versions/$VERSION.yml" +docs_src="$root/docs/$STACK" +docs_dst="$root/docs/$STACK-$TRAIN" +nav_src="$root/fern/products/$STACK/dev.yml" +nav_dst="$root/fern/products/$STACK/$TRAIN.yml" docs_yml="$root/fern/docs.yml" -catalog="$root/docs/version-catalog/main.yaml" -catalog_snapshot="$root/docs/version-catalog/$VERSION.yaml" +catalog_snapshot="$root/docs/version-catalog/$STACK-$TRAIN.yaml" [[ -d "$docs_src" ]] || { echo "Missing $docs_src" >&2; exit 1; } [[ -f "$nav_src" ]] || { echo "Missing $nav_src" >&2; exit 1; } [[ -f "$docs_yml" ]] || { echo "Missing $docs_yml" >&2; exit 1; } -[[ -f "$catalog" ]] || { echo "Missing $catalog" >&2; exit 1; } [[ ! -d "$docs_dst" ]] || { echo "Refusing to overwrite $docs_dst" >&2; exit 1; } [[ ! -f "$nav_dst" ]] || { echo "Refusing to overwrite $nav_dst" >&2; exit 1; } [[ ! -f "$catalog_snapshot" ]] || { echo "Refusing to overwrite $catalog_snapshot" >&2; exit 1; } -stack_display="$(python3 - "$catalog" "$DISPLAY" <<'PY' -from pathlib import Path -import re -import sys - -catalog_text = Path(sys.argv[1]).read_text() -display = sys.argv[2] - - -def release_set_value(name): - match = re.search(rf'^ {re.escape(name)}:\s*(\S+)\s*$', catalog_text, re.MULTILINE) - if not match: - raise SystemExit(f"release_set is missing {name}") - return match.group(1) - - -def stack_version(name): - match = re.search( - rf'^ {re.escape(name)}:\n version:\s*(\S+)\s*$', - catalog_text, - re.MULTILINE, - ) - if not match: - raise SystemExit(f"release_set is missing {name} version") - return match.group(1) - +freeze_cmd=(go run -C "$root/tools/docs-version-sync" . --target main) +if [[ -n "${DOCS_VERSION_SYNC_CMD:-}" ]]; then + read -r -a freeze_cmd <<< "$DOCS_VERSION_SYNC_CMD" +fi -if release_set_value("status") != "qualified": - raise SystemExit("release_set must be qualified before cutting versioned docs") -if release_set_value("documentation_version") != display: - raise SystemExit("release_set documentation_version does not match requested docs version") -control_version = stack_version("control-plane") -compute_version = stack_version("compute-plane") -observability_version = stack_version("observability") -print(f"CP {control_version}, Compute {compute_version}, Obs {observability_version}") -PY -)" +echo "==> Writing catalog snapshot docs/version-catalog/$STACK-$TRAIN.yaml" +"${freeze_cmd[@]}" \ + --freeze-stack "$RELEASE_SET_STACK" \ + --freeze-train "$TRAIN" +[[ -f "$catalog_snapshot" ]] || { echo "Freeze did not write $catalog_snapshot" >&2; exit 1; } -echo "==> Snapshotting docs/user -> docs/$VERSION (dereferencing symlinks)" +echo "==> Snapshotting docs/$STACK -> docs/$STACK-$TRAIN (dereferencing symlinks)" cp -RL "$docs_src" "$docs_dst" -cp "$catalog" "$catalog_snapshot" - -echo "==> Generating fern/versions/$VERSION.yml from dev.yml" -sed "s|../../docs/user/|../../docs/$VERSION/|g" "$nav_src" > "$nav_dst" - -echo "==> Updating latest and version entries in fern/docs.yml" -python3 - "$docs_yml" "$VERSION" "$DISPLAY" "$stack_display" <<'PY' -from pathlib import Path -import re -import sys - -docs_yml = Path(sys.argv[1]) -version = sys.argv[2] -display = sys.argv[3] -stack_display = sys.argv[4] - -text = docs_yml.read_text() -versions_marker = "versions:\n" -redirects_marker = "\nredirects:" - -versions_start = text.index(versions_marker) + len(versions_marker) -redirects_start = text.index(redirects_marker, versions_start) - -prefix = text[:versions_start] -versions_block = text[versions_start:redirects_start] -suffix = text[redirects_start:] - -entries = [] -current = [] -for line in versions_block.splitlines(keepends=True): - if line.startswith("- display-name:") and current: - entries.append("".join(current)) - current = [line] - elif current or line.strip(): - current.append(line) -if current: - entries.append("".join(current)) - - -def entry_display_name(entry): - match = re.search(r'^- display-name:\s*"?([^"\n]+)"?\s*$', entry, re.MULTILINE) - return match.group(1) if match else "" - - -def entry_slug(entry): - match = re.search(r'^\s+slug:\s*"?([^"\n]*)"?\s*$', entry, re.MULTILINE) - return match.group(1) if match else "" +echo "==> Generating fern/products/$STACK/$TRAIN.yml from dev.yml" +sed "s|/docs/$STACK/|/docs/$STACK-$TRAIN/|g" "$nav_src" > "$nav_dst" -latest_entry = ( - f'- display-name: "Latest ({display})"\n' - f" path: versions/{version}.yml\n" - ' slug: ""\n' -) -dev_entry = next((entry for entry in entries if entry_display_name(entry) == "dev"), ( - "- display-name: dev\n" - " path: versions/dev.yml\n" - " slug: dev\n" -)) -stable_entry = ( - f'- display-name: "{display} ({stack_display})"\n' - f" path: versions/{version}.yml\n" - f' slug: "{version}"\n' -) -redirect_entry = ( - f' - source: "/nvcf/{version}/index.html"\n' - f' destination: "/nvcf/{version}/"\n' -) +cat < "${workdir}/docs/user/overview.md" <<'EOF' -# Overview +cat > "${workdir}/docs/observability/overview.md" <<'EOF' +# Observability EOF -cat > "${workdir}/fern/versions/dev.yml" <<'EOF' +cat > "${workdir}/fern/products/observability/dev.yml" <<'EOF' navigation: - - page: Overview - path: ../../docs/user/overview.md + - page: Observability + path: ../../../docs/observability/overview.md EOF cat > "${workdir}/fern/docs.yml" <<'EOF' -versions: -- display-name: "Latest (0.5.0)" - path: versions/v0.5.yml - slug: "" -- display-name: dev - path: versions/dev.yml - slug: dev -- display-name: "0.5.0" - path: versions/v0.5.yml - slug: "v0.5" - -redirects: - - source: "/nvcf/index.html" - destination: "/nvcf/" - - source: "/nvcf/v0.5/index.html" - destination: "/nvcf/v0.5/" +products: + - display-name: Observability Stack + slug: observability + path: products/observability/dev.yml + versions: + - display-name: dev + path: products/observability/dev.yml + slug: dev EOF -cat > "${workdir}/docs/version-catalog/main.yaml" <<'EOF' -release_set: - documentation_version: cp-1.2.3-compute-1.1.0-obs-1.0.4 - status: qualified - stacks: - control-plane: - version: 1.2.3 - compute-plane: - version: 1.1.0 - observability: - version: 1.0.4 +# Stand in for the Go freeze command: record its arguments and write the +# catalog snapshot the script expects. The Go tool has its own freeze tests. +cat > "${workdir}/fake-freeze" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$@" > "${FAKE_FREEZE_ARGS}" +[[ "${FAKE_FREEZE_FAIL:-0}" == "1" ]] && exit 1 +stack=""; train="" +while [[ $# -gt 0 ]]; do + case "$1" in + --freeze-stack) stack="$2"; shift 2 ;; + --freeze-train) train="$2"; shift 2 ;; + *) shift ;; + esac +done +echo "release_set: {stacks: {${stack}: {version: ${train}.0}}}" > "${FAKE_FREEZE_OUT}" EOF +chmod +x "${workdir}/fake-freeze" + +export DOCS_VERSION_SYNC_CMD="${workdir}/fake-freeze" +export FAKE_FREEZE_ARGS="${workdir}/freeze-args.txt" +export FAKE_FREEZE_OUT="${workdir}/docs/version-catalog/observability-1.3.yaml" + +"${workdir}/tools/scripts/cut-docs-version.sh" --stack observability --train 1.3 > "${workdir}/output.txt" + +if ! grep -qx -- '--freeze-stack' "${FAKE_FREEZE_ARGS}" || ! grep -qx 'observability' "${FAKE_FREEZE_ARGS}" || ! grep -qx '1.3' "${FAKE_FREEZE_ARGS}"; then + echo "expected the freeze command to receive the stack and train" + cat "${FAKE_FREEZE_ARGS}" + exit 1 +fi + +if [[ ! -f "${workdir}/docs/observability-1.3/overview.md" ]]; then + echo "expected docs/observability to be snapshotted into docs/observability-1.3" + exit 1 +fi -"${workdir}/tools/scripts/cut-docs-version.sh" cp-1.2.3-compute-1.1.0-obs-1.0.4 > "${workdir}/output.txt" +if ! grep -q 'path: ../../../docs/observability-1.3/overview.md' "${workdir}/fern/products/observability/1.3.yml"; then + echo "expected the frozen product version file to point at the snapshot tree" + cat "${workdir}/fern/products/observability/1.3.yml" + exit 1 +fi + +if ! grep -q 'path: products/observability/1.3.yml' "${workdir}/output.txt"; then + echo "expected the script to print the fern/docs.yml versions entry" + cat "${workdir}/output.txt" + exit 1 +fi + +if grep -q 'observability-1.3' "${workdir}/fern/docs.yml"; then + echo "expected fern/docs.yml to be left for the maintainer to edit" + exit 1 +fi -if ! grep -q 'source: "/nvcf/cp-1.2.3-compute-1.1.0-obs-1.0.4/index.html"' "${workdir}/fern/docs.yml"; then - echo "expected generated docs.yml to include the new version index redirect" - cat "${workdir}/fern/docs.yml" +if "${workdir}/tools/scripts/cut-docs-version.sh" --stack observability --train 1.3 > /dev/null 2>&1; then + echo "expected a second cut of the same train to be refused" exit 1 fi -if ! grep -q 'destination: "/nvcf/cp-1.2.3-compute-1.1.0-obs-1.0.4/"' "${workdir}/fern/docs.yml"; then - echo "expected generated docs.yml to redirect the new version index to the version root" - cat "${workdir}/fern/docs.yml" +if "${workdir}/tools/scripts/cut-docs-version.sh" --stack observability --train 1.3.0 > /dev/null 2>&1; then + echo "expected a patch version to be rejected as a train" exit 1 fi -if ! grep -q 'Latest (cp-1.2.3-compute-1.1.0-obs-1.0.4)' "${workdir}/fern/docs.yml"; then - echo "expected generated docs.yml to identify all three qualified stack versions" - cat "${workdir}/fern/docs.yml" +if "${workdir}/tools/scripts/cut-docs-version.sh" --stack overview --train 1.4 > /dev/null 2>&1; then + echo "expected the unversioned overview product to be rejected" exit 1 fi -if ! cmp -s "${workdir}/docs/version-catalog/main.yaml" "${workdir}/docs/version-catalog/cp-1.2.3-compute-1.1.0-obs-1.0.4.yaml"; then - echo "expected an immutable catalog snapshot for the docs version" +# self-managed maps to the control-plane release set key. +mkdir -p "${workdir}/docs/self-managed" "${workdir}/fern/products/self-managed" +echo "# Self-managed" > "${workdir}/docs/self-managed/overview.md" +printf 'navigation:\n - page: Overview\n path: ../../../docs/self-managed/overview.md\n' > "${workdir}/fern/products/self-managed/dev.yml" +export FAKE_FREEZE_OUT="${workdir}/docs/version-catalog/self-managed-1.1.yaml" +"${workdir}/tools/scripts/cut-docs-version.sh" --stack self-managed --train 1.1 > /dev/null +if ! grep -qx 'control-plane' "${FAKE_FREEZE_ARGS}"; then + echo "expected self-managed to freeze the control-plane release set entry" + cat "${FAKE_FREEZE_ARGS}" exit 1 fi -sed -i.bak 's/status: qualified/status: development/' "${workdir}/docs/version-catalog/main.yaml" -if "${workdir}/tools/scripts/cut-docs-version.sh" cp-1.2.4-compute-1.1.0-obs-1.0.4 > "${workdir}/invalid-output.txt" 2>&1; then - echo "expected an unqualified release set to be rejected" +# A failed freeze must not leave a partial snapshot behind. +export FAKE_FREEZE_FAIL=1 +export FAKE_FREEZE_OUT="${workdir}/docs/version-catalog/observability-1.4.yaml" +if "${workdir}/tools/scripts/cut-docs-version.sh" --stack observability --train 1.4 > /dev/null 2>&1; then + echo "expected a failed freeze to abort the cut" exit 1 fi -if [[ -e "${workdir}/docs/cp-1.2.4-compute-1.1.0-obs-1.0.4" || -e "${workdir}/fern/versions/cp-1.2.4-compute-1.1.0-obs-1.0.4.yml" || -e "${workdir}/docs/version-catalog/cp-1.2.4-compute-1.1.0-obs-1.0.4.yaml" ]]; then - echo "unqualified release set created a partial documentation snapshot" +if [[ -e "${workdir}/docs/observability-1.4" || -e "${workdir}/fern/products/observability/1.4.yml" ]]; then + echo "failed freeze created a partial documentation snapshot" exit 1 fi