From b946356576f44536f7aaebeefdba564e5bf19425 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Sat, 18 Jul 2026 07:36:14 +0200 Subject: [PATCH] Route the log block through the UTF-8 file wrappers so a non-ASCII -O keeps hts-log.txt out of a mangled twin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single -O sets both path_html and path_log (htscoremain.c:411), so `httrack -O café url` sends the logs through path_log. Since argv is UTF-8, path_log holds UTF-8 bytes, but the two-file log branch created its directory and opened hts-log.txt/hts-err.txt with the ANSI structcheck()/fopen(), which on Windows read those bytes as the codepage and dropped the logs into a second, mangled caf/ directory next to the mirror. #628 fixed the same class for the mirror root; this routes the log block through structcheck_utf8()/FOPEN() /UNLINK()/fexist_utf8() so the logs land under café/ with the mirror. On a UTF-8 filesystem the wrappers are the same calls, so only Windows changes. The cache half of #630 (hts-cache still opened ANSI, via minizip's own fopen and an ANSI opendir) is a larger, separate conversion and is left for a follow-up. A local-crawl test mirrors into a single non-ASCII -O and asserts the audits read hts-log.txt from that directory; it bites on the Windows CI leg where the encoding differs. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Xavier Roche --- src/htscoremain.c | 38 +++++++++++++++----------------- tests/69_local-intl-logdir.test | 15 +++++++++++++ tests/Makefile.am | 3 ++- tests/local-crawl.sh | 39 ++++++++++++++++++++++----------- 4 files changed, 61 insertions(+), 34 deletions(-) create mode 100755 tests/69_local-intl-logdir.test diff --git a/src/htscoremain.c b/src/htscoremain.c index 4b6ed27d..678eaf97 100644 --- a/src/htscoremain.c +++ b/src/htscoremain.c @@ -2368,30 +2368,28 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) { opt->errlog = stderr; } else if (httrack_logmode >= 2) { // deux fichiers log - structcheck(StringBuff(opt->path_log)); - if (fexist - (fconcat - (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt"))) - remove(fconcat - (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt")); - if (fexist - (fconcat - (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt"))) - remove(fconcat - (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt")); + // path_log holds UTF-8 bytes (argv is UTF-8): the ANSI file calls would + // read them as the codepage and drop the logs into a mangled twin (#630). + structcheck_utf8(StringBuff(opt->path_log)); + if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), + StringBuff(opt->path_log), "hts-log.txt"))) + UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), + StringBuff(opt->path_log), "hts-log.txt")); + if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), + StringBuff(opt->path_log), "hts-err.txt"))) + UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), + StringBuff(opt->path_log), "hts-err.txt")); /* Check FS directory structure created */ - structcheck(StringBuff(opt->path_log)); + structcheck_utf8(StringBuff(opt->path_log)); - opt->log = - fopen(fconcat - (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt"), - "w"); + opt->log = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), + StringBuff(opt->path_log), "hts-log.txt"), + "w"); if (httrack_logmode == 2) - opt->errlog = - fopen(fconcat - (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt"), - "w"); + opt->errlog = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), + StringBuff(opt->path_log), "hts-err.txt"), + "w"); else opt->errlog = opt->log; if (opt->log == NULL) { diff --git a/tests/69_local-intl-logdir.test b/tests/69_local-intl-logdir.test new file mode 100755 index 00000000..f4befe30 --- /dev/null +++ b/tests/69_local-intl-logdir.test @@ -0,0 +1,15 @@ +#!/bin/bash +# +# A single non-ASCII -O sets both path_html and path_log to "café" (#630). The +# logs (hts-log.txt/hts-err.txt) must land there, not in an ANSI-mangled twin: +# on Windows path_log holds UTF-8 bytes the raw file calls read as the codepage. +# The --errors/--files/--log-found audits all grep logroot=café/hts-log.txt, so +# a log written to the twin fails them. POSIX has no twin, so this bites on the +# Windows CI leg (like test 64). The cache twin is a separate, larger fix. + +: "${top_srcdir:=..}" + +bash "$top_srcdir/tests/local-crawl.sh" --outdir-intl 'café' --errors 0 --files 5 \ + --found 'simple/basic.html' \ + --log-found 'mirror complete in' \ + httrack 'BASEURL/simple/basic.html' diff --git a/tests/Makefile.am b/tests/Makefile.am index 7a834b05..ec711dae 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -146,6 +146,7 @@ TESTS = \ 62_lang-integrity.test \ 63_webhttrack-home.test \ 64_local-intl-outdir.test \ - 65_port-siblings.test + 65_port-siblings.test \ + 69_local-intl-logdir.test CLEANFILES = check-network_sh.cache diff --git a/tests/local-crawl.sh b/tests/local-crawl.sh index 887f7753..b7a6fb90 100755 --- a/tests/local-crawl.sh +++ b/tests/local-crawl.sh @@ -48,6 +48,7 @@ key="${testdir}/server.key" tls= verbose= html_subdir= +outdir_intl= rerun= rerun_args= rerun_dead= @@ -140,6 +141,13 @@ while test "$pos" -lt "$nargs"; do pos=$((pos + 1)) html_subdir="${args[$pos]}" ;; + --outdir-intl) + # Single non-ASCII -O "$out/NAME": path_html AND path_log are NAME, so + # the logs (and the harness reads of them) go through the non-ASCII path + # (#630). Distinct from --html-subdir, which keeps path_log ASCII. + pos=$((pos + 1)) + outdir_intl="${args[$pos]}" + ;; --errors | --errors-content | --files) audit+=("${args[$pos]}" "${args[$((pos + 1))]}") pos=$((pos + 1)) @@ -216,13 +224,18 @@ test -n "$ver" || die "could not run httrack" out="${tmpdir}/crawl" mkdir "$out" || die "could not create $out" # path_html holds the mirror + index; path_log holds hts-cache/hts-log.txt. -# Default: both are "$out". With --html-subdir, path_html becomes "$out/NAME" -# (the mirror root the audits inspect) while path_log stays "$out". +# Default: both are "$out". --html-subdir moves path_html to "$out/NAME" while +# path_log (logroot) stays "$out"; --outdir-intl moves both to "$out/NAME". mirrorroot="$out" +logroot="$out" odir="$out" if test -n "$html_subdir"; then mirrorroot="${out}/${html_subdir}" odir="${mirrorroot},${out}" +elif test -n "$outdir_intl"; then + mirrorroot="${out}/${outdir_intl}" + logroot="$mirrorroot" + odir="$mirrorroot" fi # Localhost is fast; disable the rate/bandwidth safety limits but keep a # max-time backstop so a hang cannot wedge the suite. @@ -241,7 +254,7 @@ test "$crawlres" -eq 0 || ! result "httrack exited $crawlres" || { exit 1 } result "OK" -grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2 +grep -iE "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt" >&2 # --- optional second pass: re-mirror into the same dir (cache/update path) ---- if test -n "$rerun"; then @@ -260,7 +273,7 @@ if test -n "$rerun"; then # The update summary reports "files updated"; a fresh crawl never does. Assert # it so a regression that bypasses the cache (re-crawls fresh) can't pass. info "checking update used the cache" - if grep -aqE "mirror complete in .*files updated" "${out}/hts-log.txt"; then + if grep -aqE "mirror complete in .*files updated" "${logroot}/hts-log.txt"; then result "OK" else result "update pass did not report cache activity" @@ -292,7 +305,7 @@ if test -n "$rerun_dead"; then zip="${out}/hts-cache/new.zip" test -s "$zip" || die "no cache was written by the first pass" cp "$zip" "${tmpdir}/cache-before.zip" - cp "${out}/hts-log.txt" "${tmpdir}/log-before.txt" + cp "${logroot}/hts-log.txt" "${tmpdir}/log-before.txt" stop_server "$serverpid" serverpid= info "re-running httrack against the stopped server" @@ -305,7 +318,7 @@ if test -n "$rerun_dead"; then # The dead pass must have gone through the no-data rollback, not bailed out # before the mirror loop (which would leave the cache trivially untouched). info "checking the dead pass hit the rollback" - if grep -aq "No data seems to have been transferred" "${out}/hts-log.txt"; then + if grep -aq "No data seems to have been transferred" "${logroot}/hts-log.txt"; then result "OK" else result "rollback notice not found in hts-log.txt" @@ -320,7 +333,7 @@ if test -n "$rerun_dead"; then exit 1 fi # Audits below describe the healthy crawl, not the dead pass. - cp "${tmpdir}/log-before.txt" "${out}/hts-log.txt" + cp "${tmpdir}/log-before.txt" "${logroot}/hts-log.txt" fi # --- discover the single host root (127.0.0.1_ or 127.0.0.1) ----------- @@ -350,19 +363,19 @@ while test "$i" -lt "${#audit[@]}"; do --errors) i=$((i + 1)) assert_equals "checking errors" "${audit[$i]}" \ - "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")" + "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt")" ;; --errors-content) i=$((i + 1)) - total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt") + total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt") # transient network failures (statuscode -2..-6) flake on busy loopback; # the code parens are followed by " at link" or " after N retries at link" - transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${out}/hts-log.txt" || true) + transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${logroot}/hts-log.txt" || true) assert_equals "checking content errors" "${audit[$i]}" "$((total - transient))" ;; --files) i=$((i + 1)) - nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" | + nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${logroot}/hts-log.txt" | sed -e 's/.*[[:space:]]\([^ ]*\)[[:space:]]files written.*/\1/g') assert_equals "checking files" "${audit[$i]}" "$nFiles" ;; @@ -393,7 +406,7 @@ while test "$i" -lt "${#audit[@]}"; do --log-found) i=$((i + 1)) info "checking log matches ${audit[$i]}" - if grep -aqE "${audit[$i]}" "${out}/hts-log.txt"; then result "OK"; else + if grep -aqE "${audit[$i]}" "${logroot}/hts-log.txt"; then result "OK"; else result "not in log" exit 1 fi @@ -401,7 +414,7 @@ while test "$i" -lt "${#audit[@]}"; do --log-not-found) i=$((i + 1)) info "checking log lacks ${audit[$i]}" - if grep -aqE "${audit[$i]}" "${out}/hts-log.txt"; then + if grep -aqE "${audit[$i]}" "${logroot}/hts-log.txt"; then result "present in log" exit 1 else result "OK"; fi