diff --git a/src/htscharset.c b/src/htscharset.c index 4a5b5e4d..9ca955c5 100644 --- a/src/htscharset.c +++ b/src/htscharset.c @@ -373,7 +373,8 @@ char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) { return NULL; } -char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) { +HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size, + const char *charset) { const UINT cp = hts_getCodepage(charset); return hts_convertStringCPToUTF8(s, size, cp); @@ -554,7 +555,8 @@ static char *hts_convertStringCharset(const char *s, size_t size, return NULL; } -char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) { +HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size, + const char *charset) { /* Empty string ? */ if (size == 0) { return strdup(""); diff --git a/src/htscharset.h b/src/htscharset.h index 646e8d18..57a4f05b 100644 --- a/src/htscharset.h +++ b/src/htscharset.h @@ -51,8 +51,8 @@ typedef unsigned int hts_UCS4; * Convert the string "s" from charset "charset" to UTF-8. * Return NULL upon error. **/ -extern char *hts_convertStringToUTF8(const char *s, size_t size, - const char *charset); +HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size, + const char *charset); /** * Convert the string "s" from UTF-8 to charset "charset". diff --git a/src/htsserver.c b/src/htsserver.c index 6eb42430..f649244a 100644 --- a/src/htsserver.c +++ b/src/htsserver.c @@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com #include "htsnet.h" #include "htslib.h" +#include "htscharset.h" #include #include #include @@ -749,7 +750,14 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) { RUN THE SERVER */ if (strcmp((char *) adrcd, "start") == 0) { - webhttrack_main((char *) adr + p); + /* POST body is in the form's charset, not the + UTF-8 argv the engine now assumes (#629). */ + char *const cmdl = (char *) adr + p; + char *cmdlUtf8 = hts_convertStringToUTF8( + cmdl, strlen(cmdl), LANGSEL("LANGUAGE_CHARSET")); + + webhttrack_main(cmdlUtf8 != NULL ? cmdlUtf8 : cmdl); + freet(cmdlUtf8); } else { commandRunning = 0; commandEnd = 1; diff --git a/src/htsweb.c b/src/htsweb.c index bc10a206..1f0bbafa 100644 --- a/src/htsweb.c +++ b/src/htsweb.c @@ -65,6 +65,7 @@ Please visit our Website: http://www.httrack.com #include "htsserver.h" #include "htsurlport.h" #include "htsweb.h" +#include "htscharset.h" #if USE_BEGINTHREAD==0 #error fatal: no threads support @@ -156,6 +157,7 @@ int main(int argc, char *argv[]) { printf("Initializing the server..\n"); #ifdef _WIN32 + hts_argv_utf8(&argc, &argv); { WORD wVersionRequested; // requested version WinSock API WSADATA wsadata; // Windows Sockets API data diff --git a/tests/68_webhttrack-outdir-charset.test b/tests/68_webhttrack-outdir-charset.test new file mode 100755 index 00000000..332894d0 --- /dev/null +++ b/tests/68_webhttrack-outdir-charset.test @@ -0,0 +1,90 @@ +#!/bin/bash +# +# webhttrack's POST body arrives in the form's declared charset, not UTF-8, yet +# the engine now assumes UTF-8 argv. A non-ASCII -O output dir submitted through +# the web UI must land under the UTF-8 directory, not an ISO-8859-1 twin (#629). +# Drives the real htsserver over HTTP; the default (English) form is served +# ISO-8859-1, so 'café' travels as the single byte 0xE9. The crawl target is a +# dead port: only the -O directory name is under test, and htsserver creates it +# from the decoded path before any fetch. + +set -euo pipefail + +testdir=$(cd "$(dirname "$0")" && pwd) +distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)} +distdir=$(cd "${distdir}" && pwd) +# shellcheck source=tests/testlib.sh +. "${testdir}/testlib.sh" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +command -v htsserver >/dev/null || fail "no htsserver in PATH" +python=$(find_python) || { + echo "python3 not found; skipping" >&2 + exit 77 +} + +work=$(mktemp -d "${TMPDIR:-/tmp}/webhttrack_charset.XXXXXX") || fail "no tmpdir" +srvlog=$(mktemp) +srv= +cleanup() { + # htsserver keeps SIGTERM ignored across its exec, so only -9 reaps it. + test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true + wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice + rm -rf "${work}" "${srvlog}" +} +trap cleanup EXIT HUP INT QUIT PIPE TERM + +# webhttrack server on a pre-picked port; an isolated HOME keeps a stray +# ~/.httrack.ini out of it. +sport=$("${python}" -c 'import socket +s = socket.socket() +s.bind(("127.0.0.1", 0)) +print(s.getsockname()[1]) +s.close()') +( + trap '' TERM TTOU + export HOME="${work}" + exec htsserver "${distdir}/" --port "${sport}" >"${srvlog}" 2>&1 +) & +srv=$! +for _ in $(seq 1 40); do + url=$(sed -n 's/^URL=//p' "${srvlog}") && test -n "${url}" && break + kill -0 "${srv}" 2>/dev/null || break + sleep 0.25 +done +test -n "${url:-}" || fail "htsserver did not start: $(cat "${srvlog}")" + +# Post a "start" whose -O dir is 'café' in the form's ISO-8859-1 charset. +"${python}" - "${url}" "${work}" <<'PY' +import sys, urllib.parse, urllib.request + +url, work = sys.argv[1], sys.argv[2] +outdir = work + "/caf\xe9" # 'café' as the single byte the browser would send +# Port 1 refuses at once: only the decoded -O dir matters, not the fetch. +cmd = "httrack --quiet --robots=0 http://127.0.0.1:1/x.html -O " + outdir +fields = [("path", work), ("projname", "proj"), ("command_do", "start"), + ("winprofile", "x"), ("command", cmd)] +body = "&".join("%s=%s" % (k, urllib.parse.quote(v, safe="", encoding="latin-1")) + for k, v in fields) +req = urllib.request.Request(url + "step4.html", data=body.encode("latin-1"), + method="POST") +urllib.request.urlopen(req, timeout=20).read() +PY + +# The crawl runs inside htsserver; wait for the -O dir, then check it chose the +# UTF-8 name and not the ISO-8859-1 twin. +utf8dir="${work}/café" +latin1dir="${work}/caf"$'\xe9' +for _ in $(seq 1 80); do + test -e "${utf8dir}" && break + kill -0 "${srv}" 2>/dev/null || break + sleep 0.25 +done +test -e "${utf8dir}" || fail "-O dir never landed as UTF-8 café/: $(find "${work}" -maxdepth 2 2>/dev/null)" +test ! -e "${latin1dir}" || fail "-O dir created as the ISO-8859-1 twin instead" + +echo "PASS" diff --git a/tests/Makefile.am b/tests/Makefile.am index 46e91fd4..c9fc573c 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -150,6 +150,7 @@ TESTS = \ 65_port-siblings.test \ 66_engine-port80-strip.test \ 67_engine-delayed-truncate.test \ + 68_webhttrack-outdir-charset.test \ 69_local-intl-logdir.test CLEANFILES = check-network_sh.cache