diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml new file mode 100644 index 0000000..a08ac59 --- /dev/null +++ b/.github/workflows/ansible-lint.yml @@ -0,0 +1,63 @@ +--- +name: Ansible Lint + +# Actions are pinned to full commit SHAs (the trailing `# vX.Y.Z` is the tag +# that SHA corresponded to) and every tool to an exact version -- no floating +# tags. Version bumps are therefore explicit, reviewable changes to this file. +# +# Only GitHub-hosted runners and public actions are used. +# +# Soft-fail for the initial rollout: findings publish to Security -> Code +# scanning instead of blocking the branch. Tighten the `exit-code` / +# `--soft-fail` / `continue-on-error` switches once the backlog is triaged. + +on: + workflow_dispatch: + push: + branches: + - main + - release-1.0 + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + ansible-lint: + name: Ansible Lint + runs-on: ubuntu-24.04 + steps: + - name: Harden the runner (audit all outbound calls) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12.14' + + - name: Install ansible-lint + run: pip install --disable-pip-version-check "ansible-lint==26.8.0" + + # Style-only rule families are skipped: unfiltered they produced 501 + # findings here, of which 6 were substantive. name[casing] alone + # accounted for 350 of them, fighting this repo's deliberate + # `role | action | desc` naming convention. What is left is correctness + # and command-shell risk. + - name: Run ansible-lint + continue-on-error: true + run: >- + ansible-lint --nocolor + --skip-list name,var-naming,fqcn,yaml,jinja,key-order + roles/ diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml new file mode 100644 index 0000000..0a976fb --- /dev/null +++ b/.github/workflows/bandit.yml @@ -0,0 +1,94 @@ +--- +name: Bandit + +# Actions are pinned to full commit SHAs (the trailing `# vX.Y.Z` is the tag +# that SHA corresponded to) and every tool to an exact version -- no floating +# tags. Version bumps are therefore explicit, reviewable changes to this file. +# +# Only GitHub-hosted runners and public actions are used. +# +# Soft-fail for the initial rollout: findings publish to Security -> Code +# scanning instead of blocking the branch. Tighten the `exit-code` / +# `--soft-fail` / `continue-on-error` switches once the backlog is triaged. + +on: + workflow_dispatch: + push: + branches: + - main + - release-1.0 + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + schedule: + # Weekly re-scan so new findings surface without a code change. + # NOTE: GitHub only fires `schedule` on the repository's default + # branch, so this trigger stays dormant until these workflows are + # also merged to `main`. The push and pull_request triggers above + # work on every listed branch regardless. + - cron: '20 4 * * 1' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + bandit: + name: Bandit (Python security) + runs-on: ubuntu-24.04 + permissions: + contents: read + security-events: write + steps: + - name: Harden the runner (audit all outbound calls) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12.14' + + - name: Install Bandit + run: >- + pip install --disable-pip-version-check + "bandit[sarif,toml]==1.9.4" + + # Skipped tests, and why each one cannot be true here: + # B101 assert_used -- asserts in test code + # B601 paramiko_calls -- paramiko is not a dependency + # B404 import_subprocess -- importing subprocess is not a defect + # B603 subprocess_without_shell -- fires on the *recommended* argv-list + # form and cannot judge whether the input is trusted; the genuinely + # dangerous variants (B602/B604/B605/B609, shell=True and friends) + # stay enabled. + # Severity is deliberately not filtered: several real tests (e.g. B105 + # hardcoded password) report LOW, so raising the floor would hide them. + # Scanned once. The readable summary below is rendered from the SARIF + # that scan produced rather than by scanning the tree a second time. + - name: Run Bandit + run: | + bandit -r . -x ./venv,./.venv,./tests -s B101,B404,B601,B603 \ + -f sarif -o bandit-results.sarif --exit-zero + jq -r '"Bandit: \(.runs[0].results | length) finding(s)", + (.runs[0].results[] + | " \(.level)\t\(.ruleId)\t" + + "\(.locations[0].physicalLocation.artifactLocation.uri)" + + ":\(.locations[0].physicalLocation.region.startLine)\t" + + "\(.message.text | split("\n")[0])")' \ + bandit-results.sarif + + - name: Upload Bandit SARIF + uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + if: always() && hashFiles('bandit-results.sarif') != '' + with: + sarif_file: bandit-results.sarif + category: bandit diff --git a/.github/workflows/checkov.yml b/.github/workflows/checkov.yml new file mode 100644 index 0000000..49d884f --- /dev/null +++ b/.github/workflows/checkov.yml @@ -0,0 +1,87 @@ +--- +name: Checkov + +# Actions are pinned to full commit SHAs (the trailing `# vX.Y.Z` is the tag +# that SHA corresponded to) and every tool to an exact version -- no floating +# tags. Version bumps are therefore explicit, reviewable changes to this file. +# +# Only GitHub-hosted runners and public actions are used. +# +# Soft-fail for the initial rollout: findings publish to Security -> Code +# scanning instead of blocking the branch. Tighten the `exit-code` / +# `--soft-fail` / `continue-on-error` switches once the backlog is triaged. + +on: + workflow_dispatch: + push: + branches: + - main + - release-1.0 + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + schedule: + # Weekly re-scan so new findings surface without a code change. + # NOTE: GitHub only fires `schedule` on the repository's default + # branch, so this trigger stays dormant until these workflows are + # also merged to `main`. The push and pull_request triggers above + # work on every listed branch regardless. + - cron: '40 4 * * 1' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + checkov: + name: Checkov (Kubernetes / Actions / secrets) + runs-on: ubuntu-24.04 + permissions: + contents: read + security-events: write + steps: + - name: Harden the runner (audit all outbound calls) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12.14' + + - name: Install Checkov + run: pip install --disable-pip-version-check "checkov==3.3.16" + + # `helm` is deliberately not in --framework: Checkov renders charts into a + # random temp dir and reports paths relative to it, so every finding points + # at a file that does not exist in the repo -- unanchorable in the Security + # tab, and re-reported as new on every run. + # + # charts/ is skipped for the same reason the Trivy workflow skips it: it + # is vendored upstream (Istio, LiteLLM, valkey) and its defaults are not + # ours to change here. Neither tool reports on it, deliberately -- a + # decision to revisit if the charts are ever forked rather than vendored. + - name: Run Checkov + continue-on-error: true + run: | + checkov -d . \ + --framework kubernetes github_actions secrets \ + --skip-path 'charts/' \ + --soft-fail --compact --quiet \ + -o sarif --output-file-path console,. || true + ls -la results.sarif 2>/dev/null || echo "no results.sarif produced" + + - name: Upload Checkov SARIF + uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + if: always() && hashFiles('results.sarif') != '' + with: + sarif_file: results.sarif + category: checkov diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 0000000..9596b12 --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,66 @@ +--- +name: Scorecard supply-chain security + +# Actions are pinned to full commit SHAs (the trailing `# vX.Y.Z` is the tag +# that SHA corresponded to) and every tool to an exact version -- no floating +# tags. Version bumps are therefore explicit, reviewable changes to this file. +# +# Only GitHub-hosted runners and public actions are used. +# +# OpenSSF Scorecard rates repository security posture (branch protection, +# pinned dependencies, token permissions, ...). Results publish to Security -> +# Code scanning. Runs on the default branch and on a schedule -- Scorecard +# grades the repository, not an individual pull request. + +on: + workflow_dispatch: + branch_protection_rule: + push: + branches: + - main + schedule: + # Like every `schedule` trigger, this only fires once the workflow is + # present on the default branch; it is dormant on any other branch. + - cron: '45 23 * * 5' + +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-24.04 + permissions: + contents: read + security-events: write + id-token: write + steps: + - name: Harden the runner (audit all outbound calls) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4 + with: + results_file: scorecard-results.sarif + results_format: sarif + publish_results: true + + - name: Upload artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: scorecard-results + path: scorecard-results.sarif + retention-days: 5 + + - name: Upload Scorecard SARIF + uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + if: always() && hashFiles('scorecard-results.sarif') != '' + with: + sarif_file: scorecard-results.sarif + category: scorecard diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml new file mode 100644 index 0000000..1e868fe --- /dev/null +++ b/.github/workflows/shellcheck.yml @@ -0,0 +1,138 @@ +--- +name: ShellCheck + +# Actions are pinned to full commit SHAs (the trailing `# vX.Y.Z` is the tag +# that SHA corresponded to) and every tool to an exact version -- no floating +# tags. Version bumps are therefore explicit, reviewable changes to this file. +# +# Only GitHub-hosted runners and public actions are used. +# +# Soft-fail for the initial rollout: findings publish to Security -> Code +# scanning instead of blocking the branch. Tighten the `exit-code` / +# `--soft-fail` / `continue-on-error` switches once the backlog is triaged. + +on: + workflow_dispatch: + push: + branches: + - main + - release-1.0 + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + shellcheck: + name: ShellCheck (shell scripts) + runs-on: ubuntu-24.04 + steps: + - name: Harden the runner (audit all outbound calls) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # Installed from the pinned upstream release and checksum-verified rather + # than via apt, whose version drifts with the runner image. + - name: Install ShellCheck + env: + SHELLCHECK_VERSION: v0.11.0 + SHELLCHECK_SHA256: 8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198 + run: | + base="https://github.com/koalaman/shellcheck/releases/download" + file="shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" + curl -fsSL "${base}/${SHELLCHECK_VERSION}/${file}" -o shellcheck.tar.xz + echo "${SHELLCHECK_SHA256} shellcheck.tar.xz" | sha256sum --check --strict + tar -xJf shellcheck.tar.xz + sudo install -m 0755 \ + "shellcheck-${SHELLCHECK_VERSION}/shellcheck" /usr/local/bin/shellcheck + shellcheck --version + + # A `*.sh` glob missed the largest script in either tree + # (model_manager/model-manager, 1429 lines, no suffix), so discovery + # reads shebangs as well as suffixes. + # + # Entry points are linted with -x so the helper scripts they source are + # followed. The --source-path list is required because the source + # statements are built from a variable ("$MM_LIB/config.sh") that + # ShellCheck cannot resolve unaided -- without it, 7 globals that the + # helpers do set and the entry point does read looked unused. + # + # Helper scripts are then linted on their own. ShellCheck sees one file at + # a time, so a global a helper sets for its caller reads as unused: 26 of + # the 31 SC2034 hits were that. Turning SC2034 off for them was too blunt + # -- it also hid four globals that really are written and never read + # anywhere (RUNTIME_DEVICE, NRI_ADVANCED_MODE, NRI_HIDE_HT, + # NRI_TARGET_NODE). So each hit is checked individually instead, and + # suppressed only when some other file demonstrably reads that variable. + # The check applies to SHOUTY_CASE names of 3+ characters only: those are + # the cross-file globals by convention, and a short lowercase name like a + # loop's `i` collides textually with unrelated files. + - name: Run ShellCheck + continue-on-error: true + run: | + mapfile -t files < <( + git ls-files | while read -r f; do + [ -f "$f" ] || continue + case "$f" in *.sh) echo "$f"; continue ;; esac + head -1 "$f" | grep -qE '^#!.*(bash|/sh|dash|ksh)' && echo "$f" + done + ) + if [ ${#files[@]} -eq 0 ]; then + echo "No shell scripts found."; exit 0 + fi + + helpers=() entry=() + for f in "${files[@]}"; do + b=$(basename "$f") + sourcers=$(grep -lE \ + "(^|[[:space:]])(\.|source)[[:space:]]+[^|;&]*${b//./\\.}" \ + "${files[@]}" 2>/dev/null | grep -vxF "$f" || true) + if [ -n "$sourcers" ]; then helpers+=("$f"); else entry+=("$f"); fi + done + + mapfile -t dirs < <(printf '%s\n' "${files[@]}" | xargs -n1 dirname | sort -u) + sp=() + for d in "${dirs[@]}"; do sp+=(--source-path="$d"); done + + printf 'Scanning %s script(s): %s entry point(s), %s sourced helper(s)\n' \ + "${#files[@]}" "${#entry[@]}" "${#helpers[@]}" + rc=0 + + if [ ${#entry[@]} -gt 0 ]; then + shellcheck -x "${sp[@]}" --severity=error --format=gcc "${entry[@]}" || rc=1 + fi + + if [ ${#helpers[@]} -gt 0 ]; then + raw=$(shellcheck --severity=error --format=gcc "${helpers[@]}" 2>&1 || true) + kept="" + while IFS= read -r line; do + [ -n "$line" ] || continue + var=$(printf '%s\n' "$line" | sed -nE \ + 's/.*warning: ([A-Z][A-Z0-9_]{2,}) appears unused.*\[SC2034\]$/\1/p') + if [ -n "$var" ]; then + src=${line%%:*} + readers=$( + printf '%s\n' "${files[@]}" | grep -vxF "$src" | tr '\n' '\0' \ + | xargs -0 -r grep -lE "\\\$\{?${var}\b" 2>/dev/null || true + git ls-files -z | xargs -0 -r grep -lF "{{${var}}}" 2>/dev/null \ + | grep -vxF "$src" || true + ) + [ -n "$readers" ] && continue + fi + kept+="$line"$'\n' + done <<< "$raw" + if [ -n "$kept" ]; then printf '%s' "$kept"; rc=1; fi + fi + + exit "$rc" diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml new file mode 100644 index 0000000..d695e54 --- /dev/null +++ b/.github/workflows/trivy.yml @@ -0,0 +1,120 @@ +--- +name: Trivy + +# Actions are pinned to full commit SHAs (the trailing `# vX.Y.Z` is the tag +# that SHA corresponded to) and every tool to an exact version -- no floating +# tags. Version bumps are therefore explicit, reviewable changes to this file. +# +# Only GitHub-hosted runners and public actions are used. +# +# Soft-fail for the initial rollout: findings publish to Security -> Code +# scanning instead of blocking the branch. Tighten the `exit-code` / +# `--soft-fail` / `continue-on-error` switches once the backlog is triaged. + +on: + workflow_dispatch: + push: + branches: + - main + - release-1.0 + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + schedule: + # Weekly re-scan so new findings surface without a code change. + # NOTE: GitHub only fires `schedule` on the repository's default + # branch, so this trigger stays dormant until these workflows are + # also merged to `main`. The push and pull_request triggers above + # work on every listed branch regardless. + - cron: '0 4 * * 1' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + trivy: + name: Trivy (vuln, secret, misconfig, license) + runs-on: ubuntu-24.04 + permissions: + contents: read + security-events: write + steps: + - name: Harden the runner (audit all outbound calls) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # Two scans, because one severity threshold cannot serve both halves. + # CVEs, secrets and licences are triaged at CRITICAL,HIGH. Trivy rates + # nearly every Kubernetes misconfiguration check LOW or MEDIUM, so the + # same threshold would silence that scanner entirely; it instead runs at + # every severity and is scoped away from vendored charts. + # + # `limit-severities-for-sarif` must be set: it defaults to false, and + # with it unset the action drops --severity for the SARIF run only. The + # table said 0 findings while the uploaded SARIF carried 17 LOW/MEDIUM + # ones -- the Security tab silently ignored the declared threshold. + # + # NOTE: the Trivy binary is pinned, but its vulnerability database is + # fetched fresh each run -- a scanner pinned to stale CVE data is worse + # than useless. + - name: Trivy vuln / secret / licence scan (SARIF) + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + version: v0.74.0 + scan-type: fs + scan-ref: . + scanners: vuln,secret,license + severity: CRITICAL,HIGH + limit-severities-for-sarif: true + format: sarif + output: trivy-vuln.sarif + exit-code: '0' + + - name: Upload Trivy vuln SARIF + uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + if: always() && hashFiles('trivy-vuln.sarif') != '' + with: + sarif_file: trivy-vuln.sarif + category: trivy-vuln + + # charts/ is vendored upstream (Istio, LiteLLM, valkey); its defaults are + # not ours to change, and left in it contributed 16 of 17 findings. This + # is the same scoping rule the Checkov workflow applies. + - name: Trivy misconfiguration scan (SARIF) + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + version: v0.74.0 + scan-type: fs + scan-ref: . + scanners: misconfig + skip-dirs: charts + format: sarif + output: trivy-misconfig.sarif + exit-code: '0' + + - name: Upload Trivy misconfiguration SARIF + uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 + if: always() && hashFiles('trivy-misconfig.sarif') != '' + with: + sarif_file: trivy-misconfig.sarif + category: trivy-misconfig + + - name: Trivy summary (table) + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + version: v0.74.0 + scan-type: fs + scan-ref: . + scanners: vuln,secret,misconfig,license + skip-dirs: charts + format: table + exit-code: '0' diff --git a/README.md b/README.md index ff17afe..f5d8301 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ -# Intel® Enterprise for AI Inference +# Intel® AI for Enterprise Inference [![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0) -[![Component of: AI Solutions](https://img.shields.io/badge/Component%20of-AI%20Solutions-0068B5)](https://github.com/intel/enterprise-ai-solutions) +[![Component of: Intel® AI for Enterprise Solutions](https://img.shields.io/badge/Component%20of-Intel%C2%AE%20AI%20for%20Enterprise%20Solutions-0068B5)](https://github.com/intel/enterprise-ai-solutions) [![Platform: Intel Xeon](https://img.shields.io/badge/Platform-Intel%C2%AE%20Xeon%C2%AE-0068B5)](https://www.intel.com/xeon) [![Serving: KServe](https://img.shields.io/badge/Serving-KServe-326CE5)](https://kserve.github.io/website/) [![Runtimes: vLLM · OVMS](https://img.shields.io/badge/Runtimes-vLLM%20%C2%B7%20OVMS-purple)](https://vllm.ai) [![API: OpenAI Compatible](https://img.shields.io/badge/API-OpenAI%20Compatible-green)](https://platform.openai.com/docs/api-reference) -**The inference layer for Intel® Enterprise for AI Solutions. Serve any Hugging Face model on Intel® Xeon® CPUs with one command.** +**The inference layer for Intel® AI for Enterprise Solutions. Serve any Hugging Face model on Intel® Xeon® CPUs with one command.** > Provides the Ansible roles that stand up the model-serving stack — KServe, runtimes, AI gateway, LiteLLM, Langfuse — and **`model-manager`**, a CLI that takes a model from Hugging Face to a live OpenAI-compatible endpoint with NUMA-aware CPU pinning. @@ -21,7 +21,7 @@ --- -## What is Intel® Enterprise for AI Inference? +## What is Intel® AI for Enterprise Inference? Serving a model on Kubernetes normally means hand-writing manifests, sizing CPU and memory, pinning cores to the right NUMA node, downloading weights into shared storage, and wiring a route through a gateway — for every model. @@ -38,7 +38,7 @@ The result is a single OpenAI-compatible endpoint. The `model` field in the requ An inference request enters through the **LLM gateway**, which authenticates and authorizes it, then routes it to a serving engine — **vLLM** or **OpenVINO™ Model Server** — and returns an OpenAI-compatible response. The gateway also provides model endpoints, user and key management, token telemetry, and monitoring. It all runs on a Kubernetes-orchestrated, Helm-packaged stack over Intel® Xeon® infrastructure.

- Intel Enterprise for AI Inference architecture: an inference request and response enter at the top through API applications and services (samples, API apps and functions) over OpenAI-compatible API endpoints; below sits the LLM gateway providing authentication and authorization, model endpoints, user and key management, token telemetry, and monitoring; beneath it the inferencing engines vLLM and OpenVINO Model Server; then the orchestration layer with a Kubernetes orchestrator and Helm charts; and at the base the infrastructure core components — operating system (Ubuntu 22.04/24.04, RHEL) and Xeon software operators and drivers — running on Intel Xeon + Intel AI for Enterprise Inference architecture: an inference request and response enter at the top through API applications and services (samples, API apps and functions) over OpenAI-compatible API endpoints; below sits the LLM gateway providing authentication and authorization, model endpoints, user and key management, token telemetry, and monitoring; beneath it the inferencing engines vLLM and OpenVINO Model Server; then the orchestration layer with a Kubernetes orchestrator and Helm charts; and at the base the infrastructure core components — operating system (Ubuntu 22.04/24.04, RHEL) and Xeon software operators and drivers — running on Intel Xeon

> Under the hood there are two gateways: the edge gateway terminates TLS and authenticates, while the in-cluster AI gateway routes to the model the request body names. See the [Architecture deep-dive](docs/reference/architecture.md) and [Inference Request Flow](docs/reference/request_flow.md) for the full component list and execution flow. @@ -51,13 +51,13 @@ An inference request enters through the **LLM gateway**, which authenticates and > [!NOTE] > **Prerequisite:** this layer is not standalone. `es_auto_installer.sh` is the installer -> from the [**Intel® Enterprise for AI Solutions**](https://github.com/intel/enterprise-ai-solutions) +> from the [**Intel® AI for Enterprise Solutions**](https://github.com/intel/enterprise-ai-solutions) > repo, which clones this repository into `ext/enterprise.ai-inference/`. Clone that repo, prep the > machine, and create an environment, then run every command below from its root: > > ```bash > git clone https://github.com/intel/enterprise-ai-solutions.git -> cd applications.ai.enterprise.ai-solutions +> cd enterprise-ai-solutions > ./es_auto_installer.sh configure && ./es_auto_installer.sh init local > ``` > @@ -160,7 +160,7 @@ Licensed under the [Apache License, Version 2.0](https://www.apache.org/licenses - [Documentation Index](docs/README.md) - [GitHub Repository](https://github.com/intel/enterprise-inference) -- [Intel® Enterprise for AI Solutions (core platform)](https://github.com/intel/enterprise-ai-solutions) +- [Intel® AI for Enterprise Solutions (core platform)](https://github.com/intel/enterprise-ai-solutions) - [Architecture](docs/reference/architecture.md) - [Inference Request Flow](docs/reference/request_flow.md) diff --git a/docs/assets/architecture.png b/docs/assets/architecture.png index 1890ca4..6dcbe1e 100644 Binary files a/docs/assets/architecture.png and b/docs/assets/architecture.png differ diff --git a/docs/reference/architecture.md b/docs/reference/architecture.md index b4327bb..8520114 100644 --- a/docs/reference/architecture.md +++ b/docs/reference/architecture.md @@ -77,7 +77,7 @@ The core installer clones this repo to `ext/enterprise.ai-inference/` — the br set in the solutions repo's `config/repos.yaml` — and wires it in automatically: ``` -applications.ai.enterprise.ai-solutions/ +enterprise-ai-solutions/ ├── es_auto_installer.sh # main entry point ├── model-manager # thin wrapper → ext/.../model_manager/model-manager ├── config/repos.yaml # declares this repo, and which branch to clone