From 27ab7bc744f3fb01c6d44d87fa0198b306dfb29d Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Sat, 18 Jul 2026 07:27:29 +0200 Subject: [PATCH 1/2] Convert webhttrack's form-charset POST body to UTF-8 argv The httrack CLI and the WinHTTrack GUI both hand the engine UTF-8 argv, the contract htsname and htscache now assume. webhttrack did not: its back_launch_cmd splits the raw HTTP POST body into argv and calls hts_main2 directly, and that body is in the form's declared LANGUAGE_CHARSET (ISO-8859-1, windows-125x, BIG5, ... per language), never UTF-8. A non-ASCII output path or URL from the web UI reached the engine as raw form-charset bytes and was then decoded as UTF-8 downstream, landing the mirror in a mojibake directory. Convert the command line from the current LANGUAGE_CHARSET to UTF-8 before launching, at the one point where that charset is known (LANGSEL, right before webhttrack_main). ASCII and already-UTF-8 forms pass through unchanged. webhttrack's own argv also gets the Windows hts_argv_utf8 treatment the CLI has. This exports hts_convertStringToUTF8 (an additive ABI change: new symbol, soname unchanged), alongside the already-exported hts_convertStringSystemToUTF8, so the server binary can reach it. Closes #629 Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Xavier Roche --- src/htscharset.c | 6 +- src/htscharset.h | 4 +- src/htsserver.c | 10 ++- src/htsweb.c | 2 + tests/68_webhttrack-outdir-charset.test | 105 ++++++++++++++++++++++++ tests/Makefile.am | 3 +- 6 files changed, 124 insertions(+), 6 deletions(-) create mode 100755 tests/68_webhttrack-outdir-charset.test diff --git a/src/htscharset.c b/src/htscharset.c index 4a5b5e4d..9ca955c5 100644 --- a/src/htscharset.c +++ b/src/htscharset.c @@ -373,7 +373,8 @@ char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) { return NULL; } -char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) { +HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size, + const char *charset) { const UINT cp = hts_getCodepage(charset); return hts_convertStringCPToUTF8(s, size, cp); @@ -554,7 +555,8 @@ static char *hts_convertStringCharset(const char *s, size_t size, return NULL; } -char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) { +HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size, + const char *charset) { /* Empty string ? */ if (size == 0) { return strdup(""); diff --git a/src/htscharset.h b/src/htscharset.h index 646e8d18..57a4f05b 100644 --- a/src/htscharset.h +++ b/src/htscharset.h @@ -51,8 +51,8 @@ typedef unsigned int hts_UCS4; * Convert the string "s" from charset "charset" to UTF-8. * Return NULL upon error. **/ -extern char *hts_convertStringToUTF8(const char *s, size_t size, - const char *charset); +HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size, + const char *charset); /** * Convert the string "s" from UTF-8 to charset "charset". diff --git a/src/htsserver.c b/src/htsserver.c index 6eb42430..f649244a 100644 --- a/src/htsserver.c +++ b/src/htsserver.c @@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com #include "htsnet.h" #include "htslib.h" +#include "htscharset.h" #include #include #include @@ -749,7 +750,14 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) { RUN THE SERVER */ if (strcmp((char *) adrcd, "start") == 0) { - webhttrack_main((char *) adr + p); + /* POST body is in the form's charset, not the + UTF-8 argv the engine now assumes (#629). */ + char *const cmdl = (char *) adr + p; + char *cmdlUtf8 = hts_convertStringToUTF8( + cmdl, strlen(cmdl), LANGSEL("LANGUAGE_CHARSET")); + + webhttrack_main(cmdlUtf8 != NULL ? cmdlUtf8 : cmdl); + freet(cmdlUtf8); } else { commandRunning = 0; commandEnd = 1; diff --git a/src/htsweb.c b/src/htsweb.c index bc10a206..1f0bbafa 100644 --- a/src/htsweb.c +++ b/src/htsweb.c @@ -65,6 +65,7 @@ Please visit our Website: http://www.httrack.com #include "htsserver.h" #include "htsurlport.h" #include "htsweb.h" +#include "htscharset.h" #if USE_BEGINTHREAD==0 #error fatal: no threads support @@ -156,6 +157,7 @@ int main(int argc, char *argv[]) { printf("Initializing the server..\n"); #ifdef _WIN32 + hts_argv_utf8(&argc, &argv); { WORD wVersionRequested; // requested version WinSock API WSADATA wsadata; // Windows Sockets API data diff --git a/tests/68_webhttrack-outdir-charset.test b/tests/68_webhttrack-outdir-charset.test new file mode 100755 index 00000000..6777a8c1 --- /dev/null +++ b/tests/68_webhttrack-outdir-charset.test @@ -0,0 +1,105 @@ +#!/bin/bash +# +# webhttrack's POST body arrives in the form's declared charset, not UTF-8, yet +# the engine now assumes UTF-8 argv. A non-ASCII -O output dir submitted through +# the web UI must land under the correct UTF-8 directory, not an ISO-8859-1 twin +# (#629). Drives the real htsserver over HTTP; the default (English) form is +# served ISO-8859-1, so 'café' travels as the single byte 0xE9. + +set -euo pipefail + +testdir=$(cd "$(dirname "$0")" && pwd) +distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)} +distdir=$(cd "${distdir}" && pwd) +# shellcheck source=tests/testlib.sh +. "${testdir}/testlib.sh" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +command -v htsserver >/dev/null || fail "no htsserver in PATH" +python=$(find_python) || { + echo "python3 not found; skipping" >&2 + exit 77 +} + +work=$(mktemp -d "${TMPDIR:-/tmp}/webhttrack_charset.XXXXXX") || fail "no tmpdir" +origlog=$(mktemp) +srvlog=$(mktemp) +origpid= +srvpid= +cleanup() { + local pid + for pid in "${origpid}" "${srvpid}"; do + test -n "${pid}" || continue + kill -9 "${pid}" 2>/dev/null || true + wait "${pid}" 2>/dev/null || true # absorb bash's async "Killed" notice + done + rm -rf "${work}" "${origlog}" "${srvlog}" +} +trap cleanup EXIT HUP INT QUIT PIPE TERM + +# Origin server to mirror. +"${python}" "${testdir}/local-server.py" --root "${testdir}/server-root" >"${origlog}" 2>&1 & +origpid=$! +for _ in $(seq 1 40); do + oport=$(sed -n 's/^PORT //p' "${origlog}") && test -n "${oport}" && break + kill -0 "${origpid}" 2>/dev/null || break + sleep 0.25 +done +test -n "${oport:-}" || fail "origin server did not start: $(cat "${origlog}")" + +# webhttrack server; an isolated HOME keeps a stray ~/.httrack.ini out of it. +sport=$("${python}" -c 'import socket +s = socket.socket() +s.bind(("127.0.0.1", 0)) +print(s.getsockname()[1]) +s.close()') +( + trap '' TERM TTOU + export HOME="${work}" + exec htsserver "${distdir}/" --port "${sport}" >"${srvlog}" 2>&1 +) & +srvpid=$! +for _ in $(seq 1 40); do + url=$(sed -n 's/^URL=//p' "${srvlog}") && test -n "${url}" && break + kill -0 "${srvpid}" 2>/dev/null || break + sleep 0.25 +done +test -n "${url:-}" || fail "htsserver did not start: $(cat "${srvlog}")" + +# Post a "start" command whose -O dir is 'café' in the form's ISO-8859-1 charset. +"${python}" - "${url}" "${work}" "${oport}" <<'PY' +import sys, urllib.parse, urllib.request + +url, work, oport = sys.argv[1], sys.argv[2], sys.argv[3] +outdir = work + "/caf\xe9" # 'café' as ISO-8859-1 (the byte the browser would send) +cmd = ("httrack --quiet --robots=0 " + "http://127.0.0.1:%s/simple/basic.html -O %s" % (oport, outdir)) +fields = [("path", work), ("projname", "proj"), ("command_do", "start"), + ("winprofile", "x"), ("command", cmd)] +body = "&".join("%s=%s" % (k, urllib.parse.quote(v, safe="", encoding="latin-1")) + for k, v in fields) +req = urllib.request.Request(url + "step4.html", data=body.encode("latin-1"), + method="POST") +urllib.request.urlopen(req, timeout=20).read() +PY + +utf8dir="${work}/café" +latin1dir="${work}/caf"$'\xe9' + +found= +for _ in $(seq 1 160); do + test -n "$(find "${utf8dir}" -name basic.html 2>/dev/null | head -n 1)" && { + found=1 + break + } + kill -0 "${srvpid}" 2>/dev/null || break + sleep 0.25 +done +test -n "${found}" || fail "mirror never landed under UTF-8 café/: $(find "${work}" -maxdepth 3 2>/dev/null)" +test ! -e "${latin1dir}" || fail "mirror also created the ISO-8859-1 twin dir" + +echo "PASS" diff --git a/tests/Makefile.am b/tests/Makefile.am index 7a834b05..7b49044c 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -146,6 +146,7 @@ TESTS = \ 62_lang-integrity.test \ 63_webhttrack-home.test \ 64_local-intl-outdir.test \ - 65_port-siblings.test + 65_port-siblings.test \ + 68_webhttrack-outdir-charset.test CLEANFILES = check-network_sh.cache From 4f32ec41094e298b6c7fcee70d9ce1670dc04ed4 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Sat, 18 Jul 2026 08:25:14 +0200 Subject: [PATCH 2/2] Simplify the webhttrack outdir-charset test Drop the origin server: only the decoded -O directory name is under test, and htsserver creates it from the POST'd path before any fetch, so a dead crawl target (port 1) exercises the same #629 path without a second server and its poll loop. One htsserver, one poll, modelled on the sibling webhttrack tests. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Xavier Roche --- tests/68_webhttrack-outdir-charset.test | 69 ++++++++++--------------- 1 file changed, 27 insertions(+), 42 deletions(-) diff --git a/tests/68_webhttrack-outdir-charset.test b/tests/68_webhttrack-outdir-charset.test index 6777a8c1..332894d0 100755 --- a/tests/68_webhttrack-outdir-charset.test +++ b/tests/68_webhttrack-outdir-charset.test @@ -2,9 +2,11 @@ # # webhttrack's POST body arrives in the form's declared charset, not UTF-8, yet # the engine now assumes UTF-8 argv. A non-ASCII -O output dir submitted through -# the web UI must land under the correct UTF-8 directory, not an ISO-8859-1 twin -# (#629). Drives the real htsserver over HTTP; the default (English) form is -# served ISO-8859-1, so 'café' travels as the single byte 0xE9. +# the web UI must land under the UTF-8 directory, not an ISO-8859-1 twin (#629). +# Drives the real htsserver over HTTP; the default (English) form is served +# ISO-8859-1, so 'café' travels as the single byte 0xE9. The crawl target is a +# dead port: only the -O directory name is under test, and htsserver creates it +# from the decoded path before any fetch. set -euo pipefail @@ -26,32 +28,18 @@ python=$(find_python) || { } work=$(mktemp -d "${TMPDIR:-/tmp}/webhttrack_charset.XXXXXX") || fail "no tmpdir" -origlog=$(mktemp) srvlog=$(mktemp) -origpid= -srvpid= +srv= cleanup() { - local pid - for pid in "${origpid}" "${srvpid}"; do - test -n "${pid}" || continue - kill -9 "${pid}" 2>/dev/null || true - wait "${pid}" 2>/dev/null || true # absorb bash's async "Killed" notice - done - rm -rf "${work}" "${origlog}" "${srvlog}" + # htsserver keeps SIGTERM ignored across its exec, so only -9 reaps it. + test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true + wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice + rm -rf "${work}" "${srvlog}" } trap cleanup EXIT HUP INT QUIT PIPE TERM -# Origin server to mirror. -"${python}" "${testdir}/local-server.py" --root "${testdir}/server-root" >"${origlog}" 2>&1 & -origpid=$! -for _ in $(seq 1 40); do - oport=$(sed -n 's/^PORT //p' "${origlog}") && test -n "${oport}" && break - kill -0 "${origpid}" 2>/dev/null || break - sleep 0.25 -done -test -n "${oport:-}" || fail "origin server did not start: $(cat "${origlog}")" - -# webhttrack server; an isolated HOME keeps a stray ~/.httrack.ini out of it. +# webhttrack server on a pre-picked port; an isolated HOME keeps a stray +# ~/.httrack.ini out of it. sport=$("${python}" -c 'import socket s = socket.socket() s.bind(("127.0.0.1", 0)) @@ -62,22 +50,22 @@ s.close()') export HOME="${work}" exec htsserver "${distdir}/" --port "${sport}" >"${srvlog}" 2>&1 ) & -srvpid=$! +srv=$! for _ in $(seq 1 40); do url=$(sed -n 's/^URL=//p' "${srvlog}") && test -n "${url}" && break - kill -0 "${srvpid}" 2>/dev/null || break + kill -0 "${srv}" 2>/dev/null || break sleep 0.25 done test -n "${url:-}" || fail "htsserver did not start: $(cat "${srvlog}")" -# Post a "start" command whose -O dir is 'café' in the form's ISO-8859-1 charset. -"${python}" - "${url}" "${work}" "${oport}" <<'PY' +# Post a "start" whose -O dir is 'café' in the form's ISO-8859-1 charset. +"${python}" - "${url}" "${work}" <<'PY' import sys, urllib.parse, urllib.request -url, work, oport = sys.argv[1], sys.argv[2], sys.argv[3] -outdir = work + "/caf\xe9" # 'café' as ISO-8859-1 (the byte the browser would send) -cmd = ("httrack --quiet --robots=0 " - "http://127.0.0.1:%s/simple/basic.html -O %s" % (oport, outdir)) +url, work = sys.argv[1], sys.argv[2] +outdir = work + "/caf\xe9" # 'café' as the single byte the browser would send +# Port 1 refuses at once: only the decoded -O dir matters, not the fetch. +cmd = "httrack --quiet --robots=0 http://127.0.0.1:1/x.html -O " + outdir fields = [("path", work), ("projname", "proj"), ("command_do", "start"), ("winprofile", "x"), ("command", cmd)] body = "&".join("%s=%s" % (k, urllib.parse.quote(v, safe="", encoding="latin-1")) @@ -87,19 +75,16 @@ req = urllib.request.Request(url + "step4.html", data=body.encode("latin-1"), urllib.request.urlopen(req, timeout=20).read() PY +# The crawl runs inside htsserver; wait for the -O dir, then check it chose the +# UTF-8 name and not the ISO-8859-1 twin. utf8dir="${work}/café" latin1dir="${work}/caf"$'\xe9' - -found= -for _ in $(seq 1 160); do - test -n "$(find "${utf8dir}" -name basic.html 2>/dev/null | head -n 1)" && { - found=1 - break - } - kill -0 "${srvpid}" 2>/dev/null || break +for _ in $(seq 1 80); do + test -e "${utf8dir}" && break + kill -0 "${srv}" 2>/dev/null || break sleep 0.25 done -test -n "${found}" || fail "mirror never landed under UTF-8 café/: $(find "${work}" -maxdepth 3 2>/dev/null)" -test ! -e "${latin1dir}" || fail "mirror also created the ISO-8859-1 twin dir" +test -e "${utf8dir}" || fail "-O dir never landed as UTF-8 café/: $(find "${work}" -maxdepth 2 2>/dev/null)" +test ! -e "${latin1dir}" || fail "-O dir created as the ISO-8859-1 twin instead" echo "PASS"