Add a translation pipeline

.github/workflows/build_main_documentation.yml

@@ -49,6 +49,18 @@ on:
       python_version:
         type: string
         description: "Python version for the build venv (e.g. '3.11'). Defaults to the runner's system Python."
+      translation_archive:
+        type: string
+        description: "Exact completed translation archive in an HF Bucket"
+      translation_archive_sha256:
+        type: string
+        description: "Archive SHA-256 returned by the translation runner"
+      translation_language:
+        type: string
+        description: "The single translated language built by this call"
+      doc_builder_revision:
+        type: string
+        description: "Pinned doc-builder implementation SHA for a translation archive"
     secrets:
       hf_token:
         required: true
@@ -68,11 +80,13 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           repository: 'huggingface/doc-builder'
+          ref: ${{ inputs.translation_archive != '' && inputs.doc_builder_revision || '' }}
           path: doc-builder
 
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           repository: '${{ inputs.repo_owner }}/${{ inputs.package }}'
+          ref: ${{ inputs.translation_archive != '' && inputs.commit_sha || '' }}
           path: ${{ inputs.package }}
 
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -121,12 +135,16 @@ jobs:
           toolchain: stable
 
       - name: Setup environment
+        env:
+          TRANSLATION_ARCHIVE: ${{ inputs.translation_archive }}
         shell: bash
         run: |
           uv venv ${{ inputs.python_version && format('--python {0}', inputs.python_version) || '' }}
           source .venv/bin/activate
           cd doc-builder
-          git pull origin main
+          if [ -z "$TRANSLATION_ARCHIVE" ]; then
+            git pull origin main
+          fi
           uv pip install .
           cd ..
 
@@ -186,6 +204,27 @@ jobs:
 
           doc-builder notebook-to-mdx ${{ env.doc_folder }} --open_notebook_prefix https://colab.research.google.com/github/${{ inputs.repo_owner }}/${{ inputs.package }}/blob/$branch$remaining_part
 
+      - name: Install verified translation source
+        if: inputs.translation_archive != ''
+        env:
+          HF_TOKEN: ${{ secrets.hf_token }}
+          ARCHIVE: ${{ inputs.translation_archive }}
+          ARCHIVE_SHA256: ${{ inputs.translation_archive_sha256 }}
+          SOURCE_REVISION: ${{ inputs.commit_sha }}
+          BUILDER_REVISION: ${{ inputs.doc_builder_revision }}
+          LANGUAGE: ${{ inputs.translation_language }}
+          LANGUAGES: ${{ inputs.languages }}
+          PACKAGE: ${{ inputs.package }}
+          DOCS_SOURCE: ${{ env.doc_folder }}
+        run: |
+          source .venv/bin/activate
+          test "$PACKAGE" = transformers
+          test -n "$BUILDER_REVISION"
+          test "$LANGUAGE" = "$LANGUAGES"
+          python -m doc_builder.translate.artifact --archive "$ARCHIVE" --sha256 "$ARCHIVE_SHA256" \
+            --source-revision "$SOURCE_REVISION" --doc-builder-revision "$BUILDER_REVISION" \
+            --language "$LANGUAGE" --repository "$PACKAGE" --docs-source "$DOCS_SOURCE"
+
       - name: Make documentation
         shell: bash
         env:

.github/workflows/preview_translation.yml

@@ -0,0 +1,79 @@
+name: Preview translated documentation
+
+on:
+  workflow_dispatch:
+    inputs:
+      source_revision:
+        description: "Full Transformers source commit SHA"
+        required: true
+        type: string
+      language:
+        description: "Target language"
+        default: ja
+        type: string
+      pages:
+        description: "Space-separated page paths, or empty for all English pages"
+        type: string
+      bucket:
+        description: "Existing preview Bucket: hf://buckets/<owner>/<name>"
+        required: true
+        type: string
+      namespace:
+        description: "Hub namespace that runs and pays for the Job"
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: translation-transformers-${{ inputs.language }}-${{ inputs.bucket }}
+  cancel-in-progress: false
+
+jobs:
+  preview:
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    env:
+      HF_TOKEN: ${{ secrets.HF_TOKEN }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          ref: ${{ github.sha }}
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e
+        with:
+          node-version: '22.14.0'
+      - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3
+      - name: Install runner and parser
+        run: |
+          uv venv --python 3.12
+          uv pip install -e . 'huggingface_hub==1.27.0'
+          npm ci --prefix kit
+      - name: Translate on HF Jobs
+        id: translation
+        env:
+          SOURCE_REVISION: ${{ inputs.source_revision }}
+          LANGUAGE: ${{ inputs.language }}
+          BUCKET: ${{ inputs.bucket }}
+          NAMESPACE: ${{ inputs.namespace }}
+          PAGES: ${{ inputs.pages }}
+          BUILDER_REPOSITORY: https://github.com/${{ github.repository }}.git
+        run: |
+          args=()
+          if [[ -n "$PAGES" ]]; then
+            read -r -a pages <<< "$PAGES"
+            args+=(--pages "${pages[@]}")
+          fi
+          .venv/bin/python -m doc_builder.translate.job --source-revision "$SOURCE_REVISION" \
+            --lang "$LANGUAGE" --bucket "$BUCKET" --namespace "$NAMESPACE" \
+            --doc-builder-repository "$BUILDER_REPOSITORY" "${args[@]}"
+      - name: Stop an unfinished Job
+        if: always()
+        run: .venv/bin/python -m doc_builder.translate.job --cancel-record translation-job.json
+      - name: Link to translated Markdown
+        if: success()
+        env:
+          FOLDER_URL: ${{ steps.translation.outputs.folder_url }}
+          PAGE_URL: ${{ steps.translation.outputs.page_url }}
+        run: |
+          printf '[Open the completed translation folder](%s)\n\n[Read a translated page](%s)\n' "$FOLDER_URL" "$PAGE_URL" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/test.yml

@@ -21,5 +21,10 @@ jobs:
       run: |
         uv sync --extra all
         uv add git+https://github.com/huggingface/transformers
+    - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e
+      with:
+        node-version: '22.14.0'
+    - name: Install translation parser
+      run: npm ci --prefix kit
     - name: Run Tests
       run: uv run python -m pytest -n 1 --dist=loadfile -s -v ./tests/

.gitignore

@@ -15,3 +15,7 @@ build
 dist
 src/hf_doc_builder.egg-info
 src/doc_builder.egg-info
+
+# Local translation notes
+/docs/translation.md
+/plans/