webhttrack passes the engine form-charset argv instead of UTF-8

src/htscharset.c

@@ -373,7 +373,8 @@ char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) {
   return NULL;
 }
 
-char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) {
+HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
+                                         const char *charset) {
   const UINT cp = hts_getCodepage(charset);
 
   return hts_convertStringCPToUTF8(s, size, cp);
@@ -554,7 +555,8 @@ static char *hts_convertStringCharset(const char *s, size_t size,
   return NULL;
 }
 
-char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) {
+HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
+                                         const char *charset) {
   /* Empty string ? */
   if (size == 0) {
     return strdup("");

src/htscharset.h

@@ -51,8 +51,8 @@ typedef unsigned int hts_UCS4;
  * Convert the string "s" from charset "charset" to UTF-8.
  * Return NULL upon error.
  **/
-extern char *hts_convertStringToUTF8(const char *s, size_t size,
-                                     const char *charset);
+HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
+                                         const char *charset);
 
 /**
  * Convert the string "s" from UTF-8 to charset "charset".

src/htsserver.c

@@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com
 
 #include "htsnet.h"
 #include "htslib.h"
+#include "htscharset.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -749,7 +750,14 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
                            RUN THE SERVER
                          */
                         if (strcmp((char *) adrcd, "start") == 0) {
-                          webhttrack_main((char *) adr + p);
+                          /* POST body is in the form's charset, not the
+                             UTF-8 argv the engine now assumes (#629). */
+                          char *const cmdl = (char *) adr + p;
+                          char *cmdlUtf8 = hts_convertStringToUTF8(
+                              cmdl, strlen(cmdl), LANGSEL("LANGUAGE_CHARSET"));
+
+                          webhttrack_main(cmdlUtf8 != NULL ? cmdlUtf8 : cmdl);
+                          freet(cmdlUtf8);
                         } else {
                           commandRunning = 0;
                           commandEnd = 1;

src/htsweb.c

@@ -65,6 +65,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsserver.h"
 #include "htsurlport.h"
 #include "htsweb.h"
+#include "htscharset.h"
 
 #if USE_BEGINTHREAD==0
 #error fatal: no threads support
@@ -156,6 +157,7 @@ int main(int argc, char *argv[]) {
   printf("Initializing the server..\n");
 
 #ifdef _WIN32
+  hts_argv_utf8(&argc, &argv);
   {
     WORD wVersionRequested;     // requested version WinSock API
     WSADATA wsadata;            // Windows Sockets API data

tests/68_webhttrack-outdir-charset.test

@@ -0,0 +1,90 @@
+#!/bin/bash
+#
+# webhttrack's POST body arrives in the form's declared charset, not UTF-8, yet
+# the engine now assumes UTF-8 argv. A non-ASCII -O output dir submitted through
+# the web UI must land under the UTF-8 directory, not an ISO-8859-1 twin (#629).
+# Drives the real htsserver over HTTP; the default (English) form is served
+# ISO-8859-1, so 'café' travels as the single byte 0xE9. The crawl target is a
+# dead port: only the -O directory name is under test, and htsserver creates it
+# from the decoded path before any fetch.
+
+set -euo pipefail
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
+distdir=$(cd "${distdir}" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+command -v htsserver >/dev/null || fail "no htsserver in PATH"
+python=$(find_python) || {
+    echo "python3 not found; skipping" >&2
+    exit 77
+}
+
+work=$(mktemp -d "${TMPDIR:-/tmp}/webhttrack_charset.XXXXXX") || fail "no tmpdir"
+srvlog=$(mktemp)
+srv=
+cleanup() {
+    # htsserver keeps SIGTERM ignored across its exec, so only -9 reaps it.
+    test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
+    wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice
+    rm -rf "${work}" "${srvlog}"
+}
+trap cleanup EXIT HUP INT QUIT PIPE TERM
+
+# webhttrack server on a pre-picked port; an isolated HOME keeps a stray
+# ~/.httrack.ini out of it.
+sport=$("${python}" -c 'import socket
+s = socket.socket()
+s.bind(("127.0.0.1", 0))
+print(s.getsockname()[1])
+s.close()')
+(
+    trap '' TERM TTOU
+    export HOME="${work}"
+    exec htsserver "${distdir}/" --port "${sport}" >"${srvlog}" 2>&1
+) &
+srv=$!
+for _ in $(seq 1 40); do
+    url=$(sed -n 's/^URL=//p' "${srvlog}") && test -n "${url}" && break
+    kill -0 "${srv}" 2>/dev/null || break
+    sleep 0.25
+done
+test -n "${url:-}" || fail "htsserver did not start: $(cat "${srvlog}")"
+
+# Post a "start" whose -O dir is 'café' in the form's ISO-8859-1 charset.
+"${python}" - "${url}" "${work}" <<'PY'
+import sys, urllib.parse, urllib.request
+
+url, work = sys.argv[1], sys.argv[2]
+outdir = work + "/caf\xe9"  # 'café' as the single byte the browser would send
+# Port 1 refuses at once: only the decoded -O dir matters, not the fetch.
+cmd = "httrack --quiet --robots=0 http://127.0.0.1:1/x.html -O " + outdir
+fields = [("path", work), ("projname", "proj"), ("command_do", "start"),
+          ("winprofile", "x"), ("command", cmd)]
+body = "&".join("%s=%s" % (k, urllib.parse.quote(v, safe="", encoding="latin-1"))
+                for k, v in fields)
+req = urllib.request.Request(url + "step4.html", data=body.encode("latin-1"),
+                             method="POST")
+urllib.request.urlopen(req, timeout=20).read()
+PY
+
+# The crawl runs inside htsserver; wait for the -O dir, then check it chose the
+# UTF-8 name and not the ISO-8859-1 twin.
+utf8dir="${work}/café"
+latin1dir="${work}/caf"$'\xe9'
+for _ in $(seq 1 80); do
+    test -e "${utf8dir}" && break
+    kill -0 "${srv}" 2>/dev/null || break
+    sleep 0.25
+done
+test -e "${utf8dir}" || fail "-O dir never landed as UTF-8 café/: $(find "${work}" -maxdepth 2 2>/dev/null)"
+test ! -e "${latin1dir}" || fail "-O dir created as the ISO-8859-1 twin instead"
+
+echo "PASS"

tests/Makefile.am

@@ -150,6 +150,7 @@ TESTS = \
 	65_port-siblings.test \
 	66_engine-port80-strip.test \
 	67_engine-delayed-truncate.test \
+	68_webhttrack-outdir-charset.test \
 	69_local-intl-logdir.test
 
 CLEANFILES = check-network_sh.cache