|
| 1 | +#!/usr/bin/env bash |
| 2 | +# check_unicode_paths.sh: mcpp#693 on Windows: a project in a directory whose |
| 3 | +# name is not ASCII builds and runs on every toolchain row, and the files mcpp |
| 4 | +# writes for other tools hold that name in UTF-8. |
| 5 | +# |
| 6 | +# THE SHAPE OF THE FAILURE THIS GUARDS. Up to mcpp 2026.9.25.1 mcpp held paths in |
| 7 | +# the process ANSI code page. A name that page could spell (U+00E9 on code page |
| 8 | +# 1252, every Chinese name on code page 936) reached compile_commands.json as |
| 9 | +# ANSI bytes and failed every build with |
| 10 | +# error: internal: unhandled exception: [json.exception.type_error.316] |
| 11 | +# and a name it could not spell failed with a narrowing exception. mcpp.exe now |
| 12 | +# declares the UTF-8 code page, build programs are linked with the same |
| 13 | +# declaration, and the response files of the MSVC tools begin with a byte order |
| 14 | +# mark. Each of those is a separate way back to the failure, which is why every |
| 15 | +# row runs in every directory. |
| 16 | +# |
| 17 | +# Rows: llvm (clang++, the Windows default), msvc (cl.exe) and mingw (gcc@16.1.0 |
| 18 | +# for x86_64-windows-gnu), each in an ASCII directory, a directory named |
| 19 | +# `caf` + U+00E9 (inside code page 1252) and one named U+6D4B U+8BD5 (outside |
| 20 | +# it). One more project runs a build.mcpp that reads its own directory from the |
| 21 | +# environment and prints it back as an include directory: the round trip of a |
| 22 | +# path through a build program. Names are built from bytes, so this file stays |
| 23 | +# ASCII. |
| 24 | +# |
| 25 | +# Usage: MCPP=<mcpp.exe> bash .github/tools/check_unicode_paths.sh |
| 26 | +set -uo pipefail |
| 27 | + |
| 28 | +: "${MCPP:?set MCPP to the mcpp.exe under test}" |
| 29 | +ROOT=$(mktemp -d) |
| 30 | +CAFE="caf$(printf '\xc3\xa9')" |
| 31 | +CJK="$(printf '\xe6\xb5\x8b\xe8\xaf\x95')" |
| 32 | +CAFE_ANSI="caf$(printf '\xe9')" |
| 33 | +failed=0 |
| 34 | + |
| 35 | +ok() { echo " ok $*"; } |
| 36 | +bad() { echo " FAIL $*"; failed=1; } |
| 37 | + |
| 38 | +# write_project DIR TOOLCHAIN_LINES |
| 39 | +write_project() { |
| 40 | + mkdir -p "$1/src" |
| 41 | + printf '[package]\nname = "unicodeprobe"\nversion = "0.1.0"\n\n%b\n' "$2" > "$1/mcpp.toml" |
| 42 | + cat > "$1/src/main.cpp" <<'EOF' |
| 43 | +import std; |
| 44 | +int main() { |
| 45 | + std::println("unicode probe"); |
| 46 | + return 0; |
| 47 | +} |
| 48 | +EOF |
| 49 | +} |
| 50 | + |
| 51 | +# The generated file holds the directory name in UTF-8, and never in the ANSI |
| 52 | +# code page. |
| 53 | +check_bytes() { |
| 54 | + local file=$1 name=$2 row=$3 |
| 55 | + [ -f "$file" ] || { bad "$row: no $(basename "$file")"; return; } |
| 56 | + if ! LC_ALL=C grep -qF "$name" "$file"; then |
| 57 | + bad "$row: $(basename "$file") does not hold the directory name in UTF-8" |
| 58 | + fi |
| 59 | + if [ "$name" = "$CAFE" ] && LC_ALL=C grep -qF "$CAFE_ANSI" "$file"; then |
| 60 | + bad "$row: $(basename "$file") holds the ANSI spelling of the directory name" |
| 61 | + fi |
| 62 | +} |
| 63 | + |
| 64 | +rows=( |
| 65 | + 'llvm|[toolchain]\nwindows = "llvm@20.1.7"|' |
| 66 | + 'msvc|[toolchain]\nwindows = "msvc@system"|' |
| 67 | + 'mingw|[toolchain]\ndefault = "gcc@16.1.0"|--target x86_64-windows-gnu' |
| 68 | +) |
| 69 | + |
| 70 | +for name in ascii "$CAFE" "$CJK"; do |
| 71 | + for row in "${rows[@]}"; do |
| 72 | + IFS='|' read -r rid tc args <<<"$row" |
| 73 | + dir="$ROOT/$name/$rid" |
| 74 | + label="$rid in '$name'" |
| 75 | + write_project "$dir" "$tc" |
| 76 | + cd "$dir" || { bad "$label: cannot enter the directory"; continue; } |
| 77 | + # shellcheck disable=SC2086 |
| 78 | + if ! "$MCPP" build $args > build.log 2>&1; then |
| 79 | + bad "$label: the build failed"; tail -15 build.log | sed 's/^/ /'; continue |
| 80 | + fi |
| 81 | + # shellcheck disable=SC2086 |
| 82 | + out=$("$MCPP" run $args 2>&1) |
| 83 | + if ! grep -q 'unicode probe' <<<"$out"; then |
| 84 | + bad "$label: the program did not run"; printf '%s\n' "$out" | tail -5 | sed 's/^/ /' |
| 85 | + continue |
| 86 | + fi |
| 87 | + if [ "$name" != ascii ]; then |
| 88 | + ninja=$(find target -name build.ninja -newer mcpp.toml | head -1) |
| 89 | + check_bytes "$ninja" "$name" "$label" |
| 90 | + check_bytes compile_commands.json "$name" "$label" |
| 91 | + fi |
| 92 | + ok "$label" |
| 93 | + done |
| 94 | +done |
| 95 | + |
| 96 | +# A path through a build program: MCPP_MANIFEST_DIR in, `mcpp:include-dir` out, |
| 97 | +# and the header found at the directory the program printed. |
| 98 | +dir="$ROOT/$CJK/buildprogram" |
| 99 | +mkdir -p "$dir/src" "$dir/inc" |
| 100 | +printf '[package]\nname = "unicodebp"\nversion = "0.1.0"\n\n[toolchain]\nwindows = "llvm@20.1.7"\n' > "$dir/mcpp.toml" |
| 101 | +printf '#define UNICODE_BP 42\n' > "$dir/inc/unicode_bp.h" |
| 102 | +cat > "$dir/build.mcpp" <<'EOF' |
| 103 | +#include <cstdio> |
| 104 | +#include <cstdlib> |
| 105 | +int main() { |
| 106 | + const char* here = std::getenv("MCPP_MANIFEST_DIR"); |
| 107 | + if (!here) return 1; |
| 108 | + std::printf("mcpp:include-dir=%s/inc\n", here); |
| 109 | + return 0; |
| 110 | +} |
| 111 | +EOF |
| 112 | +cat > "$dir/src/main.cpp" <<'EOF' |
| 113 | +#include "unicode_bp.h" |
| 114 | +import std; |
| 115 | +int main() { |
| 116 | + std::println("build program {}", UNICODE_BP); |
| 117 | + return 0; |
| 118 | +} |
| 119 | +EOF |
| 120 | +cd "$dir" |
| 121 | +if "$MCPP" build > build.log 2>&1 && "$MCPP" run 2>&1 | grep -q 'build program 42'; then |
| 122 | + ok "a path through build.mcpp in '$CJK'" |
| 123 | +else |
| 124 | + bad "a path through build.mcpp in '$CJK'"; tail -15 build.log | sed 's/^/ /' |
| 125 | +fi |
| 126 | + |
| 127 | +cd / && rm -rf "$ROOT" |
| 128 | +[ "$failed" = 0 ] && echo "OK: every row builds in every directory" && exit 0 |
| 129 | +echo "FAIL: at least one row did not build in a directory whose name is not ASCII" |
| 130 | +exit 1 |
0 commit comments