diff --git a/.appends/.github/labels.yml b/.appends/.github/labels.yml index 9dcfdcfac6..5237dec460 100644 --- a/.appends/.github/labels.yml +++ b/.appends/.github/labels.yml @@ -75,6 +75,10 @@ description: "" color: "ededed" +- name: "ready-to-translate" + description: "The English in this PR is final; queue its translations" + color: "1d76db" + - name: "todo" description: "" color: "C6FC00" diff --git a/.github/labels.yml b/.github/labels.yml index 5882105b24..69b60dc851 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -244,6 +244,10 @@ description: "" color: "ededed" +- name: "ready-to-translate" + description: "The English in this PR is final; queue its translations" + color: "1d76db" + - name: "todo" description: "" color: "C6FC00" diff --git a/.github/workflows/i18n-completeness.yml b/.github/workflows/i18n-completeness.yml new file mode 100644 index 0000000000..c33c2e2d34 --- /dev/null +++ b/.github/workflows/i18n-completeness.yml @@ -0,0 +1,94 @@ +# Template. Installed as .github/workflows/i18n-completeness.yml in each repo +# that holds English. Keep the filename, because exercism/i18n's +# rerun-source-check.yml finds this workflow's runs by it. Don't edit a copy in +# a source repo. Edit exercism/i18n/source-repo-workflows/ and re-sync. +# +# This workflow checks whether exercism/i18n holds the translation for every +# production locale for all the English changed in a PR. If any translation is +# missing, the check fails. Once `completeness` is a required status check on +# `main`, a PR can't be merged until its translations exist. +# +# The logic lives in exercism/i18n's scripts/completeness.mjs. This workflow +# fetches what that script needs and runs it. +# +# Fork safety: +# +# - It runs on `pull_request`, so PRs from forks get a read-only token and no +# secrets. It doesn't need any, because both repos are public. +# - The PR is never checked out. Its merge ref is fetched into a bare +# repository without file contents and read with `git ls-tree` and +# `git cat-file`, so nothing from the PR is written to disk or run. +# - The only code that runs is exercism/i18n's scripts, from its `main` branch. +# - Website YAML and TypeScript files are parsed as data and never run. +# +# What counts as a change: +# +# GitHub's `refs/pull//merge` is what `main` would look like after the merge, +# and its first parent is the current `main`. The check compares the two, so a +# PR only needs translations for the English it adds or changes. If the PR has +# a merge conflict, GitHub has no merge ref and the check fails with a message +# saying so. + +name: i18n completeness + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: i18n-completeness-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + completeness: + runs-on: ubuntu-latest + steps: + # Checks out exercism/i18n for its scripts and translations. The PR's own + # repo is never checked out. + # + # TODO(iHiD): once locales/ holds a few hundred thousand files, make this a + # sparse checkout of scripts/, the two JSON files and the production + # locales only. + - name: Check out exercism/i18n (scripts and translations) + uses: actions/checkout@v7 + with: + repository: exercism/i18n + ref: main + path: i18n + persist-credentials: false + + - uses: pnpm/action-setup@v6 + with: + package_json_file: i18n/package.json + + - uses: actions/setup-node@v7 + with: + node-version: 24 + + # Installs the scripts' one dependency, `yaml`, which is only needed for + # the website. + - name: Install + run: pnpm install --frozen-lockfile --dir i18n + + - name: Fetch the PR's merge ref as git objects (no working tree) + env: + REPO: ${{ github.repository }} + PR: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + git init --quiet --bare pr + git -C pr remote add origin "https://github.com/${REPO}.git" + git -C pr config remote.origin.promisor true + git -C pr config remote.origin.partialclonefilter blob:none + if ! git -C pr fetch --quiet --depth=2 --filter=blob:none --no-tags origin "refs/pull/${PR}/merge"; then + echo "::error::This PR has no merge ref, which usually means it conflicts with main. Resolve the conflict and this check will run." + exit 1 + fi + + - name: Does exercism/i18n hold every translation this PR needs? + env: + REPO: ${{ github.repository }} + run: node i18n/scripts/completeness.mjs --source-repo=pr --repo="$REPO" --head=FETCH_HEAD --base='FETCH_HEAD^1' diff --git a/.github/workflows/i18n-queue.yml b/.github/workflows/i18n-queue.yml new file mode 100644 index 0000000000..5bda6075bb --- /dev/null +++ b/.github/workflows/i18n-queue.yml @@ -0,0 +1,355 @@ +# Template. Installed as .github/workflows/i18n-queue.yml in each repo that +# holds English. Don't edit a copy in a source repo. Edit +# exercism/i18n/source-repo-workflows/ and re-sync. +# +# This workflow queues a PR's English changes for translation by opening an +# issue in exercism/i18n. +# +# How it works: +# +# 1. A PR changes English. +# 2. When the copy is final, a maintainer adds the `ready-to-translate` label. +# Nothing is queued without it, whoever opened the PR. Only people with +# triage rights can add labels. The label must exist in this repo (see +# "Before installing" in exercism/i18n/source-repo-workflows/README.md). +# 3. This workflow opens an issue in exercism/i18n that lists the changed +# files, their blob ids and the PR's head commit. +# 4. The translator translates the files, pushes them to exercism/i18n main +# and closes the issue. Closing the issue re-runs the PR's +# `i18n completeness` check, which then passes. +# +# The i18n issue records each step. The PR gets one reply, from +# exercism/i18n's rerun-source-check.yml, once the translations have landed and +# the check has re-run, so the maintainer knows the PR can be merged. +# +# Jobs: +# +# queue Runs when the label is added, and opens the issue (or updates the +# open one). It also runs after a push that changed no English, to +# move an open issue to the new head commit. A rebase or force-push +# can remove the old commit from the PR, and the translator only +# accepts commits that are in the PR. +# hold Runs on each push while the label is on. If the push changed +# English, it removes the label and asks a maintainer to add it +# again once the copy is final. +# withdraw Runs when the label is removed, and closes the open issue as +# "not planned". It also runs straight after `hold`, because a label +# removed with GITHUB_TOKEN doesn't trigger an `unlabeled` run. +# +# Fork safety: +# +# The workflow uses `pull_request_target` because it needs secrets and a token +# that can edit labels, and fork PRs get neither under `pull_request`. These +# runs have this repo's tokens, so checking out or running PR code would expose +# them to the PR's author. To prevent that: +# +# - The workflow never checks out this repository. It only checks out +# exercism/i18n at main, for its scripts. +# - The workflow reads the changed files from the GitHub API (PR files, +# compare, trees and blobs) and treats them as data. It never clones or runs +# anything from the PR. +# - exercism/i18n's scripts/english-changes.mjs validates the API responses +# and escapes paths before they go into Markdown. The workflow passes the PR +# title only as an environment variable, and the issue body as a file. +# - Each job's GITHUB_TOKEN has only the permissions that job needs. The +# jobs that read the PR's files have read access only, except `hold`, which +# removes the label. The exercism/i18n token is a separate secret. +# +# exercism/i18n defines which files count as English, in +# scripts/lib/content-types.mjs and scripts/lib/website-english.mjs. +# english-changes.mjs reads those lists, so this workflow keeps no copy of them. + +name: i18n queue + +on: + pull_request_target: + types: [labeled, unlabeled, synchronize] + branches: [main] + +permissions: {} + +jobs: + queue: + needs: hold + # Needs `always()` because `hold` is skipped on `labeled` events. + if: ${{ always() && ((github.event.action == 'labeled' && github.event.label.name == 'ready-to-translate') || (github.event.action == 'synchronize' && needs.hold.outputs.english-changed == 'false')) }} + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + # If the label is removed and added again quickly, only the latest run + # continues. Pushes use a separate group, so a push straight after the label + # is added cannot cancel the run that opens the issue. + concurrency: + group: i18n-queue-${{ github.event.pull_request.number }}-${{ github.event.action }} + cancel-in-progress: true + steps: + # Checks out exercism/i18n only. This repository is never checked out. + - name: Check out exercism/i18n (scripts only) + uses: actions/checkout@v7 + with: + repository: exercism/i18n + ref: main + path: i18n + persist-credentials: false + sparse-checkout: | + scripts + locales.json + website-exclusions.json + + - uses: actions/setup-node@v7 + with: + node-version: 24 + + - name: Which English changed + id: changed + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/files?per_page=100" > files.json + # The base commit's tree (from the API), used to find the old blob id + # of each changed config.json or metadata.toml. + gh api "repos/${REPO}/git/trees/${BASE_SHA}?recursive=1" > base-tree.json + + # First pass: list the blobs needed to work out which keys changed. + node i18n/scripts/english-changes.mjs --repo="$REPO" --pr-files=files.json --base-tree=base-tree.json --needs=needs.txt > /dev/null + + # needs.txt holds only 40-character hex ids, and the loop checks each + # line again. The blobs are fetched through the API, saved under their + # id, and only parsed as JSON or flat TOML. + mkdir -p blobs + while read -r sha; do + [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || continue + gh api "repos/${REPO}/git/blobs/${sha}" --jq .content | base64 -d > "blobs/${sha}" || rm -f "blobs/${sha}" + done < needs.txt + + # Second pass: the full result. If a blob could not be fetched, its + # file is still reported as changed, without the list of keys. + node i18n/scripts/english-changes.mjs --repo="$REPO" --pr-files=files.json --base-tree=base-tree.json --blobs-dir=blobs --markdown=changes.md | tee out.txt + grep '^count=' out.txt >> "$GITHUB_OUTPUT" + + - name: Open or update the i18n issue + id: issue + if: steps.changed.outputs.count != '0' + env: + # An organisation secret on `exercism`, available to every repo. It is + # a fine-grained PAT owned by iHiD with read/write access to issues on + # exercism/i18n and nothing else, so the issues are authored by iHiD. + GH_TOKEN: ${{ secrets.EXERCISM_I18N_ISSUES_PAT }} + # This repo's token, used only to re-read the PR's labels. + READ_TOKEN: ${{ github.token }} + I18N_REPO: exercism/i18n + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_TITLE: ${{ github.event.pull_request.title }} + PR_URL: ${{ github.event.pull_request.html_url }} + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + COUNT: ${{ steps.changed.outputs.count }} + ACTION: ${{ github.event.action }} + run: | + set -euo pipefail + + # Check the label is still on. `hold` may have removed it since this + # run started. + labelled() { + GH_TOKEN="$READ_TOKEN" gh api "repos/${REPO}/pulls/${PR_NUMBER}" \ + --jq '[.labels[].name] | index("ready-to-translate") != null' + } + if [ "$(labelled)" != "true" ]; then + echo "::notice::ready-to-translate came off before the issue was written; nothing was queued." + exit 0 + fi + + # Limit the title's length and keep it on one line. + SAFE_TITLE=$(printf '%s' "$PR_TITLE" | tr -d '\r\n' | cut -c1-120) + PREFIX="Translate ${REPO}#${PR_NUMBER}:" + TITLE="${PREFIX} ${SAFE_TITLE}" + + { + printf 'English changed in %s, by @%s, and a maintainer marked it `ready-to-translate`.\n\n' "$PR_URL" "$PR_AUTHOR" + printf '| | |\n|---|---|\n' + printf '| Repo | %s |\n' "$REPO" + printf '| Translate at | %s |\n' "$HEAD_SHA" + printf '| Base | %s |\n' "$BASE_SHA" + printf '| English files changed | %s |\n\n' "$COUNT" + printf '**Read English at %s, not at main.** A push to the PR that changes English takes the label off and closes this issue as not planned; putting the label back opens a new one.\n\n' "$HEAD_SHA" + cat changes.md + printf '\n---\n\n' + printf 'Close this when the translations are on `main` here for every locale in `locales.json` `productionTargets`. Closing it re-runs the `i18n completeness` check on the PR, which is what is holding its merge.\n\n' + printf 'Add and update only, never delete: `main` over there still defines the old text until the PR merges, and every other open PR is checked against what is here.\n' + } > body.md + + EXISTING=$(gh issue list --repo "$I18N_REPO" --state open --label translation --search "in:title \"${PREFIX}\"" --limit 50 \ + --json number,title | PREFIX="$PREFIX" jq -r 'first(.[] | select(.title | startswith(env.PREFIX)) | .number) // empty') + + # After a push with no English changes, only update an issue that is + # already open. + if [ "$ACTION" = "synchronize" ] && [ -z "$EXISTING" ]; then + echo "No open issue for ${REPO}#${PR_NUMBER}; nothing to move to ${HEAD_SHA}." + exit 0 + fi + + if [ -n "$EXISTING" ]; then + gh issue edit "$EXISTING" --repo "$I18N_REPO" --title "$TITLE" --body-file body.md + if [ "$ACTION" = "synchronize" ]; then + gh issue comment "$EXISTING" --repo "$I18N_REPO" --body "The PR's head moved with no change to English. Translate at ${HEAD_SHA}." + else + gh issue comment "$EXISTING" --repo "$I18N_REPO" --body "Re-queued. Translate at ${HEAD_SHA}." + fi + NUMBER="$EXISTING" + else + NUMBER=$(gh issue create --repo "$I18N_REPO" --title "$TITLE" --body-file body.md --label translation | sed -E 's#.*/issues/([0-9]+).*#\1#') + fi + echo "Wrote ${I18N_REPO}#${NUMBER} at ${HEAD_SHA}." + + # Check the label again after writing. `hold` removes the label before + # looking for an issue to close, so whichever job runs second sees the + # other's change, and no issue is left open without the label. + if [ "$(labelled)" = "false" ] && [[ "$NUMBER" =~ ^[1-9][0-9]*$ ]]; then + gh issue close "$NUMBER" --repo "$I18N_REPO" --reason "not planned" \ + --comment "Withdrawn: \`ready-to-translate\` came off ${REPO}#${PR_NUMBER} while this was being written." + fi + + hold: + if: ${{ github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'ready-to-translate') }} + runs-on: ubuntu-latest + # No concurrency group. Each push is checked on its own `before`..`after`, + # and GitHub cancels pending runs in a group even with cancel-in-progress + # off, which could miss a push that changed English. If two runs remove the + # label at once, only one removal succeeds. + permissions: + contents: read + pull-requests: write + outputs: + english-changed: ${{ steps.changed.outputs.count != '0' }} + steps: + - name: Check out exercism/i18n (scripts only) + uses: actions/checkout@v7 + with: + repository: exercism/i18n + ref: main + path: i18n + persist-credentials: false + sparse-checkout: | + scripts + locales.json + website-exclusions.json + + - uses: actions/setup-node@v7 + with: + node-version: 24 + + # Checks whether this push changed English in the PR's files. As in + # `queue`, every input comes from the API. If anything cannot be read, the + # push counts as a change, since the only cost is a maintainer adding the + # label again. + - name: Did this push change English + id: changed + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.event.after }} + run: | + set -euo pipefail + unknown() { + echo "::warning::$1 Counting this push as changing English." + echo "count=unknown" >> "$GITHUB_OUTPUT" + exit 0 + } + [[ "$BEFORE" =~ ^[0-9a-f]{40}$ && "$AFTER" =~ ^[0-9a-f]{40}$ ]] || unknown "The event names no before and after commit." + + gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/files?per_page=100" > files.json || unknown "Could not list the PR's files." + # Lists the files the PR changed as of `before`, so English that this + # push removed from the PR still counts. + gh api "repos/${REPO}/compare/${BASE_SHA}...${BEFORE}" --jq '.files' > before-files.json || unknown "Could not compare the base with ${BEFORE}." + gh api "repos/${REPO}/git/trees/${BEFORE}?recursive=1" > before-tree.json || unknown "Could not read the tree at ${BEFORE}." + gh api "repos/${REPO}/git/trees/${AFTER}?recursive=1" > after-tree.json || unknown "Could not read the tree at ${AFTER}." + + ARGS=(--push --repo="$REPO" --pr-files=files.json --before-files=before-files.json --before-tree=before-tree.json --after-tree=after-tree.json) + node i18n/scripts/english-changes.mjs "${ARGS[@]}" --needs=needs.txt > /dev/null || unknown "english-changes.mjs failed." + + # As in `queue`: hex ids only, fetched as data, parsed as JSON or TOML. + mkdir -p blobs + while read -r sha; do + [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || continue + gh api "repos/${REPO}/git/blobs/${sha}" --jq .content | base64 -d > "blobs/${sha}" || rm -f "blobs/${sha}" + done < needs.txt + + node i18n/scripts/english-changes.mjs "${ARGS[@]}" --blobs-dir=blobs > out.txt || unknown "english-changes.mjs failed." + cat out.txt + grep '^count=' out.txt >> "$GITHUB_OUTPUT" + + - name: Take the label off and say why + if: steps.changed.outputs.count != '0' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + AFTER: ${{ github.event.after }} + run: | + set -euo pipefail + # If the label is already gone, another run or a person removed it, so + # there is no comment to add. + if ! gh api -X DELETE "repos/${REPO}/issues/${PR_NUMBER}/labels/ready-to-translate" > /dev/null; then + echo "The label was already off; no comment." + exit 0 + fi + printf '%s\n' \ + "This push changed English, so \`ready-to-translate\` has come off and any queued translation is withdrawn." \ + "" \ + "A maintainer: please re-apply \`ready-to-translate\` once the copy is final. That queues the translation at the PR's head commit." \ + > comment.md + gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" -F body=@comment.md > /dev/null + echo "Took ready-to-translate off ${REPO}#${PR_NUMBER} at ${AFTER}." + + withdraw: + needs: hold + # Needs `always()` because `hold` is skipped on `unlabeled` events. + if: ${{ always() && ((github.event.action == 'unlabeled' && github.event.label.name == 'ready-to-translate') || needs.hold.outputs.english-changed == 'true') }} + runs-on: ubuntu-latest + permissions: + pull-requests: read + steps: + # Closing as "not planned" means exercism/i18n's rerun-source-check.yml + # ignores it, and the translator's retry sweep (which lists open issues + # only) skips it. An open issue would be translated at a commit whose + # English is out of date. + - name: Close the open i18n issue as not planned + env: + GH_TOKEN: ${{ secrets.EXERCISM_I18N_ISSUES_PAT }} + READ_TOKEN: ${{ github.token }} + I18N_REPO: exercism/i18n + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + WHY: ${{ github.event.action == 'unlabeled' && 'someone took it off by hand' || 'a push changed English' }} + run: | + set -euo pipefail + # If the label has been added back, leave the issue to `queue`. Closing + # it here could close the issue `queue` has just written. + ON=$(GH_TOKEN="$READ_TOKEN" gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '[.labels[].name] | index("ready-to-translate") != null') + if [ "$ON" = "true" ]; then + echo "ready-to-translate is on ${REPO}#${PR_NUMBER} again; leaving its issue to the queue job." + exit 0 + fi + PREFIX="Translate ${REPO}#${PR_NUMBER}:" + NUMBERS=$(gh issue list --repo "$I18N_REPO" --state open --label translation --search "in:title \"${PREFIX}\"" --limit 50 \ + --json number,title | PREFIX="$PREFIX" jq -r '.[] | select(.title | startswith(env.PREFIX)) | .number') + if [ -z "$NUMBERS" ]; then + echo "No open issue for ${REPO}#${PR_NUMBER}; nothing to withdraw." + exit 0 + fi + for number in $NUMBERS; do + [[ "$number" =~ ^[1-9][0-9]{0,8}$ ]] || continue + gh issue close "$number" --repo "$I18N_REPO" --reason "not planned" \ + --comment "Withdrawn: \`ready-to-translate\` came off ${REPO}#${PR_NUMBER} because ${WHY}. Re-applying the label opens a new issue at the PR's head." + echo "Closed ${I18N_REPO}#${number}." + done