From a3bfa2dc9d529e413834529fda1bffa35c0cdde2 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Sun, 12 Jul 2026 18:37:47 +0200 Subject: [PATCH 1/5] native: 64-bit multi-ABI + bump httrack engine 3.49-2 -> 3.49.12 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the JNI native build to 64-bit and the current engine. Validated locally with NDK r26d (Clang 17): libiconv.so, libhttrack.so (OpenSSL 3.0.15 statically linked), libhtslibjni.so all build for arm64-v8a AND armeabi-v7a. Android.mk: - per-ABI prebuilt OpenSSL (../prebuild/$(TARGET_ARCH_ABI)/lib{ssl,crypto}.a) + per-ABI openssl include dir - drop hardcoded TARGET_ARCH := arm - gate armv7-only -Wl,--no-merge-exidx-entries behind armeabi-v7a (it breaks the arm64/x86_64 link) - libiconv-1.15 -> libiconv-1.17 - add new engine library sources (htssniff.c, hts*_selftest.c) — required by the 3.49.12 engine; drop the vestigial libhtsjava module (engine removed htsjava.c) Application.mk: APP_ABI = arm64-v8a armeabi-v7a x86_64; APP_PLATFORM android-24 (bionic introduces minizip's fseeko/ftello at API 24 — min-API decision noted). config.h: SIZEOF_LONG/SIZEOF_LONG_LONG compiler-derived (was hardcoded 4 — wrong on 64-bit ABIs). HTTrackLib.java: drop 'htsjava' from the native load list. Submodule httrack -> 627ae2b (3.49.12), which already selects TLS_client_method() for OpenSSL 3.x. Requires 'git submodule update --init --recursive' (coucal is nested). x86_64 is deferred pending a coffeecatch x86_64 ucontext fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 6 ++ .../com/httrack/android/jni/HTTrackLib.java | 2 +- app/src/main/jni/Android.mk | 74 +++++++++---------- app/src/main/jni/Application.mk | 11 ++- app/src/main/jni/httrack | 2 +- app/src/main/jni/include/config.h | 9 ++- 6 files changed, 56 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index 3cc136d..20210a6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,9 @@ workspace.xml /app/.externalNativeBuild /app/build /app/src/main/jniLibs + +# Build inputs fetched/generated by tools/ci (not committed): +# libiconv vendored by tools/ci/vendor-libiconv.sh +# per-ABI OpenSSL statics from the toolchain image / fetch-openssl-statics.sh +/app/src/main/jni/libiconv-1.17/ +/app/src/main/prebuild/ diff --git a/app/src/main/java/com/httrack/android/jni/HTTrackLib.java b/app/src/main/java/com/httrack/android/jni/HTTrackLib.java index 7841b70..01ef9df 100755 --- a/app/src/main/java/com/httrack/android/jni/HTTrackLib.java +++ b/app/src/main/java/com/httrack/android/jni/HTTrackLib.java @@ -191,7 +191,7 @@ public static boolean loadLibraries() { try { /** Load all libraries. **/ - final String[] libraries = { "iconv", "httrack", "htsjava", "htslibjni" }; + final String[] libraries = { "iconv", "httrack", "htslibjni" }; for (final String lib : libraries) { Log.d(TAG, "Loading native library " + lib); System.loadLibrary(lib); diff --git a/app/src/main/jni/Android.mk b/app/src/main/jni/Android.mk index da43005..e845757 100755 --- a/app/src/main/jni/Android.mk +++ b/app/src/main/jni/Android.mk @@ -14,7 +14,15 @@ # LOCAL_PATH := $(call my-dir) -TARGET_ARCH := arm + +# The exidx-merge linker flag is armv7-only (ARM EXIDX sections exist only on +# 32-bit ARM); emitting it on arm64-v8a/x86_64 breaks the link. Gate it per-ABI. +# ndk-build re-includes this file once per ABI with TARGET_ARCH_ABI set. +ifeq ($(TARGET_ARCH_ABI),armeabi-v7a) + HTS_ARMV7_LDFLAGS := -Wl,--no-merge-exidx-entries +else + HTS_ARMV7_LDFLAGS := +endif # include $(CLEAR_VARS) @@ -27,17 +35,17 @@ LOCAL_CFLAGS := \ -DIN_LIBRARY \ -DLIBICONV_PLUG LOCAL_SRC_FILES := \ - libiconv-1.15/libcharset/lib/localcharset.c \ - libiconv-1.15/lib/iconv.c \ - libiconv-1.15/lib/relocatable.c + libiconv-1.17/libcharset/lib/localcharset.c \ + libiconv-1.17/lib/iconv.c \ + libiconv-1.17/lib/relocatable.c LOCAL_C_INCLUDES += \ $(LOCAL_PATH)/include/iconv \ - $(LOCAL_PATH)/libiconv-1.15/include \ - $(LOCAL_PATH)/libiconv-1.15/libcharset \ - $(LOCAL_PATH)/libiconv-1.15/lib \ - $(LOCAL_PATH)/libiconv-1.15/libcharset/include \ - $(LOCAL_PATH)/libiconv-1.15/srclib -LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)/libiconv-1.15/include + $(LOCAL_PATH)/libiconv-1.17/include \ + $(LOCAL_PATH)/libiconv-1.17/libcharset \ + $(LOCAL_PATH)/libiconv-1.17/lib \ + $(LOCAL_PATH)/libiconv-1.17/libcharset/include \ + $(LOCAL_PATH)/libiconv-1.17/srclib +LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)/libiconv-1.17/include include $(BUILD_SHARED_LIBRARY) # TODO FIXME: INVESTIGATE THIS @@ -52,12 +60,12 @@ include $(BUILD_SHARED_LIBRARY) include $(CLEAR_VARS) LOCAL_MODULE := libcrypto -LOCAL_SRC_FILES := ../prebuild/libcrypto.a +LOCAL_SRC_FILES := ../prebuild/$(TARGET_ARCH_ABI)/libcrypto.a include $(PREBUILT_STATIC_LIBRARY) include $(CLEAR_VARS) LOCAL_MODULE := libssl -LOCAL_SRC_FILES := ../prebuild/libssl.a +LOCAL_SRC_FILES := ../prebuild/$(TARGET_ARCH_ABI)/libssl.a include $(PREBUILT_STATIC_LIBRARY) include $(CLEAR_VARS) @@ -72,14 +80,18 @@ LOCAL_SRC_FILES := httrack/src/htscore.c httrack/src/htsparse.c \ httrack/src/htsmd5.c httrack/src/htszlib.c httrack/src/htswrap.c \ httrack/src/htsconcat.c httrack/src/htsmodules.c \ httrack/src/htscharset.c httrack/src/punycode.c \ - httrack/src/htsencoding.c httrack/src/md5.c httrack/src/minizip/ioapi.c \ + httrack/src/htsencoding.c httrack/src/md5.c \ + httrack/src/htssniff.c httrack/src/htsselftest.c \ + httrack/src/htscache_selftest.c httrack/src/htsdns_selftest.c \ + httrack/src/minizip/ioapi.c \ httrack/src/minizip/mztools.c httrack/src/minizip/unzip.c \ httrack/src/minizip/zip.c LOCAL_C_INCLUDES := $(LOCAL_PATH)/httrack/src \ $(LOCAL_PATH)/httrack/src/coucal \ + $(LOCAL_PATH)/../prebuild/$(TARGET_ARCH_ABI)/include \ $(LOCAL_PATH)/include LOCAL_LDLIBS := -ldl -lz -LOCAL_LDLIBS += -L$(LOCAL_PATH)/../prebuild +LOCAL_LDLIBS += -L$(LOCAL_PATH)/../prebuild/$(TARGET_ARCH_ABI) LOCAL_SHARED_LIBRARIES := libiconv LOCAL_STATIC_LIBRARIES := libssl libcrypto LOCAL_CFLAGS += -O3 -g3 -funwind-tables -fPIC -rdynamic \ @@ -96,35 +108,14 @@ LOCAL_CFLAGS += -O3 -g3 -funwind-tables -fPIC -rdynamic \ -D_REENTRANT -DPIC -DANDROID -D_ANDROID -DHAVE_CONFIG_H -DINET6 \ -DLIBHTTRACK_EXPORTS -DZLIB_CONST -DHTS_INTHASH_USES_MD5 -DLIBICONV_PLUG\ -DHTS_CRASH_TEST \ - -Wl,--no-merge-exidx-entries -Wl,-O1 + $(HTS_ARMV7_LDFLAGS) -Wl,-O1 LOCAL_CPPFLAGS += -pthread include $(BUILD_SHARED_LIBRARY) -include $(CLEAR_VARS) -LOCAL_MODULE := libhtsjava -LOCAL_SRC_FILES := httrack/src/htsjava.c -LOCAL_C_INCLUDES := $(LOCAL_PATH)/httrack/src \ - $(LOCAL_PATH)/httrack/src/coucal \ - $(LOCAL_PATH)/include -LOCAL_LDLIBS += -L$(LOCAL_PATH)/../prebuild -LOCAL_SHARED_LIBRARIES := libhttrack -LOCAL_CFLAGS += -O3 -g3 -funwind-tables -fPIC -rdynamic \ - -fstrict-aliasing -fvisibility=hidden \ - -Wall -Wformat -Wformat-security -Wmultichar -Wwrite-strings -Wcast-qual\ - -Wcast-align -Wstrict-prototypes -Wmissing-prototypes \ - -Wmissing-declarations -Wdeclaration-after-statement -Wpointer-arith \ - -Wsequence-point -Wnested-externs -Wparentheses -Winit-self \ - -Wunused-but-set-parameter -Waddress -Wuninitialized -Wformat=2 \ - -Wformat-nonliteral -Wmissing-parameter-type -Wold-style-definition \ - -Wignored-qualifiers -Wstrict-aliasing -Wno-sign-compare \ - -Wno-type-limits -Wno-missing-field-initializers -Wno-cast-align \ - -Wno-nested-externs \ - -D_REENTRANT -DPIC -DANDROID -D_ANDROID -DHAVE_CONFIG_H -DINET6 \ - -DLIBHTTRACK_EXPORTS -DZLIB_CONST -DHTS_INTHASH_USES_MD5 -DLIBICONV_PLUG\ - -DHTS_CRASH_TEST \ - -Wl,--no-merge-exidx-entries -Wl,-O1 -LOCAL_CPPFLAGS += -pthread -include $(BUILD_SHARED_LIBRARY) +# NOTE: the old `libhtsjava` module (httrack/src/htsjava.c) was dropped — the +# engine removed htsjava.c (the desktop Java-applet parser bridge) after 3.49-2, +# and nothing on Android used it. "htsjava" is likewise removed from +# HTTrackLib.loadLibraries(). include $(CLEAR_VARS) LOCAL_MODULE := htslibjni @@ -132,10 +123,11 @@ LOCAL_SRC_FILES := htslibjni.c coffeecatch/coffeecatch.c coffeecatch/coffeejni.c LOCAL_C_INCLUDES := $(LOCAL_PATH)/httrack/src \ $(LOCAL_PATH)/httrack/src/coucal \ $(LOCAL_PATH)/coffeecatch \ + $(LOCAL_PATH)/../prebuild/$(TARGET_ARCH_ABI)/include \ $(LOCAL_PATH)/include LOCAL_SHARED_LIBRARIES := libhttrack LOCAL_LDLIBS := -llog LOCAL_CFLAGS := -O3 -g -funwind-tables \ - -Wl,--no-merge-exidx-entries -Wl,-O1 \ + $(HTS_ARMV7_LDFLAGS) -Wl,-O1 \ -W -Wall -Wextra -Werror -Wno-unused-parameter include $(BUILD_SHARED_LIBRARY) diff --git a/app/src/main/jni/Application.mk b/app/src/main/jni/Application.mk index 6d00145..d9bef21 100755 --- a/app/src/main/jni/Application.mk +++ b/app/src/main/jni/Application.mk @@ -1 +1,10 @@ -APP_ABI := armeabi-v7a +# 64-bit multi-ABI: arm64-v8a is the primary target, armeabi-v7a kept for old +# devices, x86_64 for emulators/CI. 64-bit ABIs require API 21+. +# +# APP_PLATFORM is android-24, not 21: httrack's bundled minizip (ioapi.c) calls +# fseeko/ftello for large-file (>2GB) zip-cache seeks, and bionic only introduces +# them at API 24. Keeping the min at 21 would require an upstream minizip guard +# (fall back to fseek/ftell below 24, capping the cache at 2GB). See the +# android-modernization plan — this is a min-API decision for Xavier. +APP_ABI := arm64-v8a armeabi-v7a x86_64 +APP_PLATFORM := android-24 diff --git a/app/src/main/jni/httrack b/app/src/main/jni/httrack index e8bdccc..627ae2b 160000 --- a/app/src/main/jni/httrack +++ b/app/src/main/jni/httrack @@ -1 +1 @@ -Subproject commit e8bdccc443f80ea5d0ce6dff5916625bdd81504e +Subproject commit 627ae2b04331ca9df54100c336558d857dcc8f0e diff --git a/app/src/main/jni/include/config.h b/app/src/main/jni/include/config.h index 87e8c9c..f950fef 100755 --- a/app/src/main/jni/include/config.h +++ b/app/src/main/jni/include/config.h @@ -105,11 +105,12 @@ /* Check for setuid */ #define SETUID 1 -/* The size of `long', as computed by sizeof. */ -#define SIZEOF_LONG 4 +/* The size of `long'/`long long', as computed by sizeof. This config.h is + hand-maintained and shared across ABIs, so DON'T hardcode 4 here: sizeof(long) + is 8 on arm64-v8a and x86_64. Derive it from the compiler instead. */ +#define SIZEOF_LONG (sizeof(long)) -/* The size of `long long', as computed by sizeof. */ -#define SIZEOF_LONG_LONG 8 +#define SIZEOF_LONG_LONG (sizeof(long long)) /* Define to 1 if you have the ANSI C header files. */ #define STDC_HEADERS 1 From 8479c725b79a802bd07bdc0ef7f3677e25e262e5 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Sun, 12 Jul 2026 18:38:02 +0200 Subject: [PATCH 2/5] ci: bootstrap Android GitHub Actions (Phase A) + reproducible toolchain Greenfield CI (repo had no .github). Matches the httrack repo conventions (contents: read, concurrency cancel, checkout@v6 + submodules recursive, CodeQL security-extended). - docker/: reproducible toolchain image (JDK17 + SDK35 + NDK r26d + OpenSSL 3.x statics for all ABIs). wget (curl's threaded resolver fails on some Docker hosts); build-essential for libiconv ./configure. - tools/ci/vendor-libiconv.sh: fetch + configure libiconv 1.17, patch the host-configured HAVE_LANGINFO_CODESET (bionic nl_langinfo is API 26+). - tools/ci/fetch-openssl-statics.sh: stage per-ABI OpenSSL into prebuild/. - .github/workflows/docker-image.yml: build+push the image to GHCR (+ statics artifact). - .github/workflows/android.yml: 'native' ndk-build job (arm64-v8a + armeabi-v7a; x86_64 pending coffeecatch fix). assemble/emulator jobs documented for later phases. - .github/workflows/codeql.yml + codeql-config.yml: java-kotlin + c-cpp, c-cpp scoped to htslibjni*.c (engine scanned by its own repo). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/codeql/codeql-config.yml | 14 +++++++ .github/workflows/android.yml | 60 ++++++++++++++++++++++++++++ .github/workflows/codeql.yml | 55 ++++++++++++++++++++++++++ .github/workflows/docker-image.yml | 48 +++++++++++++++++++++++ docker/Dockerfile | 63 ++++++++++++++++++++++++++++++ docker/build-openssl.sh | 59 ++++++++++++++++++++++++++++ tools/ci/fetch-openssl-statics.sh | 30 ++++++++++++++ tools/ci/vendor-libiconv.sh | 51 ++++++++++++++++++++++++ 8 files changed, 380 insertions(+) create mode 100644 .github/codeql/codeql-config.yml create mode 100644 .github/workflows/android.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/docker-image.yml create mode 100644 docker/Dockerfile create mode 100755 docker/build-openssl.sh create mode 100755 tools/ci/fetch-openssl-statics.sh create mode 100755 tools/ci/vendor-libiconv.sh diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml new file mode 100644 index 0000000..55e7c9f --- /dev/null +++ b/.github/codeql/codeql-config.yml @@ -0,0 +1,14 @@ +# Scope the c-cpp CodeQL analysis to httrack-android's OWN native code (the JNI +# glue). The vendored httrack engine, coucal, coffeecatch, and libiconv are scanned +# by their own repos, so excluding them here avoids duplicate findings/build time. +name: httrack-android CodeQL config + +paths: + - app/src/main/jni/htslibjni.c + - app/src/main/jni/htslibjni.h + - app/src/main/jni/htslibjni-private.h + +paths-ignore: + - app/src/main/jni/httrack + - app/src/main/jni/coffeecatch + - app/src/main/jni/libiconv-1.17 diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml new file mode 100644 index 0000000..0f3f5ec --- /dev/null +++ b/.github/workflows/android.yml @@ -0,0 +1,60 @@ +# Android build + test for httrack-android. +# Style follows the httrack repo (least-privilege, concurrency cancel, submodules +# recursive). Uses the toolchain image published by docker-image.yml (JDK17 + SDK +# + NDK + OpenSSL statics) so PRs skip the OpenSSL cross-compile. +name: Android + +on: + push: + branches: [master] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: android-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Native-only: proves recursive submodules (incl. the nested coucal), libiconv + # vendoring, OpenSSL 3.x, and 64-bit multi-ABI ndk-build — independently of the + # Gradle/AGP migration. Validated locally (arm64-v8a + armeabi-v7a). + # x86_64 is intentionally omitted until coffeecatch gains an x86_64 ucontext + # branch (its legacy fallback #errors on x86_64); needed for emulator-smoke. + native: + name: native (ndk-build) + runs-on: ubuntu-24.04 + container: + image: ghcr.io/xroche/httrack-android-build:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive # MUST be recursive: coucal is nested in httrack + - name: Vendor libiconv + run: tools/ci/vendor-libiconv.sh + - name: Stage prebuilt OpenSSL statics (from image) + run: tools/ci/fetch-openssl-statics.sh # reads $OPENSSL_ANDROID_ROOT in the image + env: + ABIS: arm64-v8a armeabi-v7a + - name: ndk-build + working-directory: app/src/main # contains jni/ — ndk-build auto-discovers Android.mk + Application.mk + run: ndk-build APP_ABI="arm64-v8a armeabi-v7a" APP_PLATFORM=android-24 -j"$(nproc)" + - uses: actions/upload-artifact@v4 + with: + name: native-libs + path: app/src/main/libs/**/*.so + if-no-files-found: warn + + # --- Phases below land with the app modernization (see the plan) --- + # + # assemble: ./gradlew assembleDebug assembleRelease lint — needs the AGP 8 / + # AndroidX migration (Phase 3). Add when it lands. + # + # emulator-smoke: reactivecircus/android-emulator-runner (x86_64, api 21/29/34), + # installs the APK and runs an instrumented crawl against a local fixture + # server. Needs Phase 4 (storage) + the coffeecatch x86_64 fix. diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..92d1045 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,55 @@ +# CodeQL static analysis: the Java app AND the JNI/C glue. Mirrors the httrack +# repo's codeql.yml (security-extended). c-cpp is scoped to htslibjni*.c via +# .github/codeql/codeql-config.yml so the vendored engine isn't double-scanned. +name: CodeQL + +on: + push: + branches: [master] + pull_request: + schedule: + - cron: "31 4 * * 1" + +permissions: + contents: read + +concurrency: + group: codeql-${{ github.ref }} + cancel-in-progress: true + +jobs: + analyze: + name: analyze (${{ matrix.language }}) + runs-on: ubuntu-24.04 + permissions: + contents: read + security-events: write + strategy: + fail-fast: false + matrix: + language: [java-kotlin, c-cpp] + container: + image: ghcr.io/xroche/httrack-android-build:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive + - uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + queries: security-extended + config-file: ./.github/codeql/codeql-config.yml + # c-cpp: CodeQL must observe the native compile (autobuild can't guess the + # ndk-build invocation). java-kotlin: autobuild is fine. + - name: Build native (c-cpp) + if: matrix.language == 'c-cpp' + run: | + tools/ci/vendor-libiconv.sh + ABIS=arm64-v8a tools/ci/fetch-openssl-statics.sh + ( cd app/src/main/jni && ndk-build APP_ABI=arm64-v8a APP_PLATFORM=android-24 -j"$(nproc)" ) + - uses: github/codeql-action/autobuild@v3 + if: matrix.language == 'java-kotlin' + - uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 0000000..93df4ff --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,48 @@ +# Build & publish the toolchain image (JDK17 + SDK + NDK + OpenSSL 3.x statics for +# all ABIs) to GHCR, so android.yml/codeql.yml pull a ready image instead of +# recompiling OpenSSL (~10-20 min) every run. First publish is private even for a +# public repo — set the package visibility to public once in repo Packages settings. +name: Toolchain image + +on: + push: + branches: [master] + paths: ['docker/**', '.github/workflows/docker-image.yml'] + schedule: + - cron: "23 5 * * 1" # weekly: pick up OpenSSL patch releases + workflow_dispatch: + +permissions: + contents: read + packages: write + +concurrency: + group: image-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-push: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/build-push-action@v6 + with: + context: docker + push: true + tags: | + ghcr.io/xroche/httrack-android-build:latest + ghcr.io/xroche/httrack-android-build:${{ github.sha }} + - name: Extract OpenSSL statics (for the container-less emulator job) + run: | + id=$(docker create ghcr.io/xroche/httrack-android-build:latest) + docker cp "$id":/opt/openssl-android ./openssl-android + docker rm "$id" + - uses: actions/upload-artifact@v4 + with: + name: openssl-android-statics + path: openssl-android diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..62ba0c4 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,63 @@ +# Reproducible build environment for the modernized HTTrack Android app. +# +# - JDK 17 (required by AGP 8.x) +# - Android cmdline-tools + platform 35 + build-tools 35 +# - NDK r26d (Clang, unified toolchain — replaces the r14/GCC hack) +# - OpenSSL 3.x static libs cross-compiled for arm64-v8a/armeabi-v7a/x86_64 +# +# Build: docker build -t httrack-android-build docker/ +# Use: docker run --rm -v "$PWD":/src -w /src httrack-android-build \ +# ./gradlew assembleRelease +# +# The OpenSSL libs are baked into the image under /opt/openssl-android//. +# Point Android.mk's prebuilt paths at them (or copy into app/src/main/prebuild). +FROM eclipse-temurin:17-jdk-jammy + +ARG ANDROID_CMDLINE_VERSION=11076708 +ARG ANDROID_PLATFORM=android-35 +ARG ANDROID_BUILD_TOOLS=35.0.0 +ARG NDK_VERSION=26.3.11579264 +ARG OPENSSL_VERSION=3.0.15 +ARG OPENSSL_SHA256=23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533 + +ENV ANDROID_SDK_ROOT=/opt/android-sdk \ + ANDROID_HOME=/opt/android-sdk \ + DEBIAN_FRONTEND=noninteractive + +# NOTE: use wget, not curl, for downloads. On this host's Docker/kernel combo +# curl's threaded DNS resolver fails ("getaddrinfo() thread failed to start") +# even under seccomp=unconfined, while wget's synchronous resolver works. +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget unzip git make perl file ca-certificates build-essential && \ + rm -rf /var/lib/apt/lists/* +# build-essential: needed by tools/ci/vendor-libiconv.sh (host ./configure) and +# by OpenSSL's build. wget (not curl): curl's threaded resolver fails on some +# Docker/kernel combos ("getaddrinfo() thread failed to start"). + +# --- Android SDK command-line tools --- +RUN mkdir -p "$ANDROID_SDK_ROOT/cmdline-tools" && cd /tmp && \ + wget -q "https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_CMDLINE_VERSION}_latest.zip" && \ + unzip -q "commandlinetools-linux-${ANDROID_CMDLINE_VERSION}_latest.zip" && \ + mv cmdline-tools "$ANDROID_SDK_ROOT/cmdline-tools/latest" && \ + rm "commandlinetools-linux-${ANDROID_CMDLINE_VERSION}_latest.zip" +ENV PATH="$ANDROID_SDK_ROOT/cmdline-tools/latest/bin:$ANDROID_SDK_ROOT/platform-tools:$PATH" + +# --- SDK packages + NDK --- +RUN yes | sdkmanager --licenses >/dev/null && \ + sdkmanager --install \ + "platform-tools" \ + "platforms;${ANDROID_PLATFORM}" \ + "build-tools;${ANDROID_BUILD_TOOLS}" \ + "ndk;${NDK_VERSION}" >/dev/null +ENV ANDROID_NDK_ROOT="$ANDROID_SDK_ROOT/ndk/${NDK_VERSION}" + +# --- OpenSSL 3.x static libs for all ABIs --- +COPY build-openssl.sh /usr/local/bin/build-openssl.sh +RUN chmod +x /usr/local/bin/build-openssl.sh && \ + OPENSSL_VERSION="${OPENSSL_VERSION}" OPENSSL_SHA256="${OPENSSL_SHA256}" \ + OUT=/opt/openssl-android MIN_API=21 \ + /usr/local/bin/build-openssl.sh && \ + rm -rf /tmp/* +ENV OPENSSL_ANDROID_ROOT=/opt/openssl-android + +WORKDIR /src diff --git a/docker/build-openssl.sh b/docker/build-openssl.sh new file mode 100755 index 0000000..d982b79 --- /dev/null +++ b/docker/build-openssl.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# Cross-compile OpenSSL 3.x static libs for Android, all target ABIs, using the +# NDK's own Clang toolchain. Replaces the dead 2017 setenv-android.sh + OpenSSL +# 1.0.2k hack (see ../openssl-ndk-analysis.md). +# +# Output layout (consumed by app/src/main/jni/Android.mk): +# $OUT//lib/libssl.a +# $OUT//lib/libcrypto.a +# $OUT//include/openssl/*.h +# +# Requires: ANDROID_NDK_ROOT set, wget, perl, make. Run inside the Docker image. +# (wget, not curl: curl's threaded resolver fails on this host's Docker/kernel.) +set -euo pipefail + +OPENSSL_VERSION="${OPENSSL_VERSION:-3.0.15}" +# Pin the hash — verify against https://www.openssl.org/source/ before bumping. +OPENSSL_SHA256="${OPENSSL_SHA256:-23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533}" +MIN_API="${MIN_API:-21}" +OUT="${OUT:-/opt/openssl-android}" +ABIS="${ABIS:-arm64-v8a armeabi-v7a x86_64}" + +: "${ANDROID_NDK_ROOT:?set ANDROID_NDK_ROOT}" +HOST_TAG="linux-x86_64" +export PATH="$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/$HOST_TAG/bin:$PATH" + +abi_to_target() { + case "$1" in + arm64-v8a) echo android-arm64 ;; + armeabi-v7a) echo android-arm ;; + x86_64) echo android-x86_64 ;; + x86) echo android-x86 ;; + *) echo "unknown ABI: $1" >&2; exit 1 ;; + esac +} + +work="$(mktemp -d)" +cd "$work" +wget -q "https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/openssl-${OPENSSL_VERSION}.tar.gz" +echo "${OPENSSL_SHA256} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c - +tar xf "openssl-${OPENSSL_VERSION}.tar.gz" +src="$work/openssl-${OPENSSL_VERSION}" + +for abi in $ABIS; do + target="$(abi_to_target "$abi")" + echo "=== Building OpenSSL ${OPENSSL_VERSION} for ${abi} (${target}) ===" + builddir="$work/build-$abi" + cp -a "$src" "$builddir" + ( cd "$builddir" + ./Configure "$target" "-D__ANDROID_API__=${MIN_API}" \ + no-shared no-tests no-ui-console no-engine no-comp no-dso no-legacy \ + --prefix="$OUT/$abi" --libdir=lib + make -j"$(nproc)" build_libs + make install_dev + ) +done + +echo "=== OpenSSL static libs installed under $OUT// ===" +find "$OUT" -name 'lib*.a' -printf '%p\n' diff --git a/tools/ci/fetch-openssl-statics.sh b/tools/ci/fetch-openssl-statics.sh new file mode 100755 index 0000000..25358f9 --- /dev/null +++ b/tools/ci/fetch-openssl-statics.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# Stage per-ABI OpenSSL static libs + headers into app/src/main/prebuild//, +# the layout Android.mk expects (../prebuild/$(TARGET_ARCH_ABI)/lib{ssl,crypto}.a +# and .../include/openssl). +# +# Source of the statics, in priority order: +# 1. $OPENSSL_ANDROID_ROOT (set in the toolchain Docker image → /opt/openssl-android) +# 2. build them here from a standalone NDK (needs $ANDROID_NDK_ROOT), via +# docker/build-openssl.sh — used by the container-less emulator job. +set -euo pipefail + +JNI_DIR="$(cd "$(dirname "$0")/../../app/src/main/jni" && pwd)" +PREBUILD="$JNI_DIR/../prebuild" +ABIS="${ABIS:-arm64-v8a armeabi-v7a x86_64}" +SRC="${OPENSSL_ANDROID_ROOT:-}" + +if [ -z "$SRC" ] || [ ! -d "$SRC" ]; then + echo "OPENSSL_ANDROID_ROOT not set/found — building OpenSSL from NDK ..." + : "${ANDROID_NDK_ROOT:?set ANDROID_NDK_ROOT or OPENSSL_ANDROID_ROOT}" + SRC="${OUT:-/tmp/openssl-android}" + OUT="$SRC" ABIS="$ABIS" bash "$(dirname "$0")/../../docker/build-openssl.sh" +fi + +for abi in $ABIS; do + mkdir -p "$PREBUILD/$abi/include" + cp "$SRC/$abi/lib/libssl.a" "$SRC/$abi/lib/libcrypto.a" "$PREBUILD/$abi/" + cp -r "$SRC/$abi/include/openssl" "$PREBUILD/$abi/include/" +done +echo "staged OpenSSL statics for: $ABIS" diff --git a/tools/ci/vendor-libiconv.sh b/tools/ci/vendor-libiconv.sh new file mode 100755 index 0000000..6b502ee --- /dev/null +++ b/tools/ci/vendor-libiconv.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# Vendor GNU libiconv into app/src/main/jni/libiconv-1.17/ and generate the +# headers the ndk-build needs (iconv.h from iconv.h.in, config.h). bionic has no +# iconv, so httrack bundles it (built with -DLIBICONV_PLUG). Only 3 source files +# are compiled by Android.mk; iconv.c is a mega-TU that #includes every converter, +# so it needs configure-generated headers + the tarball's pre-generated aliases.h. +# +# Idempotent: re-running with the tree already present is a no-op. Needs a host C +# compiler + make for ./configure (header generation only; the actual cross-build +# is done by ndk-build). +set -euo pipefail + +VERSION="${LIBICONV_VERSION:-1.17}" +SHA256="${LIBICONV_SHA256:-8f74213b56238c85a50a5329f77e06198771e70dd9a739779f4c02f65d971313}" +JNI_DIR="$(cd "$(dirname "$0")/../../app/src/main/jni" && pwd)" +DEST="$JNI_DIR/libiconv-${VERSION}" + +if [ -f "$DEST/include/iconv.h" ] && [ -f "$DEST/config.h" ]; then + echo "libiconv ${VERSION} already vendored + configured at $DEST" + exit 0 +fi + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +cd "$work" +echo "=== fetching libiconv ${VERSION} ===" +wget -q "https://ftp.gnu.org/gnu/libiconv/libiconv-${VERSION}.tar.gz" +echo "${SHA256} libiconv-${VERSION}.tar.gz" | sha256sum -c - +tar xf "libiconv-${VERSION}.tar.gz" + +echo "=== configure (host) to generate iconv.h + config.h ===" +( cd "libiconv-${VERSION}" && ./configure --enable-static --disable-shared >/dev/null ) + +rm -rf "$DEST" +mkdir -p "$(dirname "$DEST")" +mv "libiconv-${VERSION}" "$DEST" + +# The host ./configure detects HAVE_LANGINFO_CODESET (true on Linux), but bionic +# only introduces nl_langinfo() at API 26. localcharset.c is built without +# -DHAVE_CONFIG_H and reads whichever config.h is on the include path, so neutralize +# the define in ALL generated config.h. Android is always UTF-8, so the non- +# nl_langinfo path is correct. (Cross-config skew; a proper NDK cross-configure +# would set this automatically.) +for f in "$DEST/config.h" "$DEST/libcharset/config.h" "$DEST/lib/config.h"; do + [ -f "$f" ] && sed -i \ + 's|^#define HAVE_LANGINFO_CODESET 1|/* #undef HAVE_LANGINFO_CODESET (bionic nl_langinfo is API 26+) */|' "$f" +done + +echo "=== vendored at $DEST ===" +ls "$DEST/include/iconv.h" "$DEST/config.h" "$DEST/lib/aliases.h" 2>&1 From b90a8cdf04830387638d75d45e19e5b732c16c9a Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Sun, 12 Jul 2026 19:01:10 +0200 Subject: [PATCH 3/5] ci: make Phase A jobs self-contained (no pre-published image) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The container-based jobs failed with 'manifest unknown' — the GHCR toolchain image can't be bootstrapped from a PR branch (workflow_dispatch needs the workflow on the default branch). Rework native + codeql to set up the NDK, cross-compile OpenSSL (cached), vendor libiconv, and ndk-build inline — the flow validated locally. java-kotlin CodeQL uses build-mode: none (the app doesn't build under modern Gradle until the AGP 8 migration). docker/ image + docker-image.yml stay for the later Gradle/assemble phase. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/android.yml | 55 +++++++++++++++++++++++------------ .github/workflows/codeql.yml | 39 +++++++++++++++++-------- 2 files changed, 63 insertions(+), 31 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 0f3f5ec..83ccae8 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -1,7 +1,9 @@ # Android build + test for httrack-android. # Style follows the httrack repo (least-privilege, concurrency cancel, submodules -# recursive). Uses the toolchain image published by docker-image.yml (JDK17 + SDK -# + NDK + OpenSSL statics) so PRs skip the OpenSSL cross-compile. +# recursive). Phase A is self-contained: it sets up the NDK, cross-compiles OpenSSL +# (cached), vendors libiconv, and ndk-builds the native libs — no pre-published +# image needed. (The docker/ toolchain image + docker-image.yml exist for the later +# Gradle/assemble phase, which needs the full SDK.) name: Android on: @@ -17,33 +19,48 @@ concurrency: group: android-${{ github.ref }} cancel-in-progress: true +env: + NDK_VERSION: r26d + OPENSSL_VERSION: "3.0.15" + jobs: - # Native-only: proves recursive submodules (incl. the nested coucal), libiconv - # vendoring, OpenSSL 3.x, and 64-bit multi-ABI ndk-build — independently of the - # Gradle/AGP migration. Validated locally (arm64-v8a + armeabi-v7a). - # x86_64 is intentionally omitted until coffeecatch gains an x86_64 ucontext - # branch (its legacy fallback #errors on x86_64); needed for emulator-smoke. + # Native-only: proves recursive submodules (incl. nested coucal), libiconv + # vendoring, OpenSSL 3.x, and 64-bit multi-ABI ndk-build. Validated locally for + # arm64-v8a + armeabi-v7a. x86_64 is added once the coffeecatch x86_64 fix lands + # (xroche/coffeecatch#46) — its legacy ucontext block #errors on x86_64. native: name: native (ndk-build) runs-on: ubuntu-24.04 - container: - image: ghcr.io/xroche/httrack-android-build:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + env: + ABIS: arm64-v8a armeabi-v7a steps: - uses: actions/checkout@v6 with: submodules: recursive # MUST be recursive: coucal is nested in httrack + - name: Set up NDK ${{ env.NDK_VERSION }} + id: ndk + uses: nttld/setup-ndk@v1 + with: + ndk-version: ${{ env.NDK_VERSION }} + local-cache: true + - name: Cache OpenSSL statics + id: ssl-cache + uses: actions/cache@v4 + with: + path: /tmp/openssl-android + key: openssl-${{ env.OPENSSL_VERSION }}-ndk-${{ env.NDK_VERSION }}-${{ env.ABIS }} + - name: Build OpenSSL (cache miss) + if: steps.ssl-cache.outputs.cache-hit != 'true' + env: + ANDROID_NDK_ROOT: ${{ steps.ndk.outputs.ndk-path }} + run: OUT=/tmp/openssl-android OPENSSL_VERSION="${OPENSSL_VERSION}" bash docker/build-openssl.sh - name: Vendor libiconv run: tools/ci/vendor-libiconv.sh - - name: Stage prebuilt OpenSSL statics (from image) - run: tools/ci/fetch-openssl-statics.sh # reads $OPENSSL_ANDROID_ROOT in the image - env: - ABIS: arm64-v8a armeabi-v7a + - name: Stage OpenSSL statics into prebuild/ + run: OPENSSL_ANDROID_ROOT=/tmp/openssl-android tools/ci/fetch-openssl-statics.sh - name: ndk-build - working-directory: app/src/main # contains jni/ — ndk-build auto-discovers Android.mk + Application.mk - run: ndk-build APP_ABI="arm64-v8a armeabi-v7a" APP_PLATFORM=android-24 -j"$(nproc)" + working-directory: app/src/main # contains jni/ — ndk-build auto-discovers Android.mk + Application.mk + run: ${{ steps.ndk.outputs.ndk-path }}/ndk-build APP_ABI="${ABIS}" APP_PLATFORM=android-24 -j"$(nproc)" - uses: actions/upload-artifact@v4 with: name: native-libs @@ -53,7 +70,7 @@ jobs: # --- Phases below land with the app modernization (see the plan) --- # # assemble: ./gradlew assembleDebug assembleRelease lint — needs the AGP 8 / - # AndroidX migration (Phase 3). Add when it lands. + # AndroidX migration (Phase 3), and can use the GHCR toolchain image. # # emulator-smoke: reactivecircus/android-emulator-runner (x86_64, api 21/29/34), # installs the APK and runs an instrumented crawl against a local fixture diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 92d1045..ee21e13 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,6 +1,10 @@ # CodeQL static analysis: the Java app AND the JNI/C glue. Mirrors the httrack # repo's codeql.yml (security-extended). c-cpp is scoped to htslibjni*.c via # .github/codeql/codeql-config.yml so the vendored engine isn't double-scanned. +# +# java-kotlin uses build-mode: none (the app doesn't build under modern Gradle +# until the AGP 8 migration in Phase 3 — no build needed to analyze the source). +# c-cpp builds the JNI glue via a self-contained NDK setup (no pre-published image). name: CodeQL on: @@ -17,6 +21,10 @@ concurrency: group: codeql-${{ github.ref }} cancel-in-progress: true +env: + NDK_VERSION: r26d + OPENSSL_VERSION: "3.0.15" + jobs: analyze: name: analyze (${{ matrix.language }}) @@ -27,12 +35,9 @@ jobs: strategy: fail-fast: false matrix: - language: [java-kotlin, c-cpp] - container: - image: ghcr.io/xroche/httrack-android-build:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + include: + - { language: java-kotlin, build-mode: none } + - { language: c-cpp, build-mode: manual } steps: - uses: actions/checkout@v6 with: @@ -40,16 +45,26 @@ jobs: - uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} queries: security-extended config-file: ./.github/codeql/codeql-config.yml - # c-cpp: CodeQL must observe the native compile (autobuild can't guess the - # ndk-build invocation). java-kotlin: autobuild is fine. + + # c-cpp only: CodeQL must observe the native compile (arm64-v8a is enough). + - name: Set up NDK ${{ env.NDK_VERSION }} + if: matrix.language == 'c-cpp' + id: ndk + uses: nttld/setup-ndk@v1 + with: + ndk-version: ${{ env.NDK_VERSION }} + local-cache: true - name: Build native (c-cpp) if: matrix.language == 'c-cpp' + env: + ANDROID_NDK_ROOT: ${{ steps.ndk.outputs.ndk-path }} run: | + OUT=/tmp/openssl-android ABIS=arm64-v8a OPENSSL_VERSION="${OPENSSL_VERSION}" bash docker/build-openssl.sh tools/ci/vendor-libiconv.sh - ABIS=arm64-v8a tools/ci/fetch-openssl-statics.sh - ( cd app/src/main/jni && ndk-build APP_ABI=arm64-v8a APP_PLATFORM=android-24 -j"$(nproc)" ) - - uses: github/codeql-action/autobuild@v3 - if: matrix.language == 'java-kotlin' + OPENSSL_ANDROID_ROOT=/tmp/openssl-android ABIS=arm64-v8a tools/ci/fetch-openssl-statics.sh + ( cd app/src/main && "${ANDROID_NDK_ROOT}/ndk-build" APP_ABI=arm64-v8a APP_PLATFORM=android-24 -j"$(nproc)" ) + - uses: github/codeql-action/analyze@v3 From 8fe52be0172f372fc125e79846c3e616815f7465 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Sun, 12 Jul 2026 19:09:48 +0200 Subject: [PATCH 4/5] ci: scope CodeQL paths to include Java sources (fix java-kotlin analysis) codeql-config.yml's `paths` applies to every language, so restricting it to the htslibjni.* C files left the java-kotlin analysis with zero source ('no source code seen'). Add app/src/main/java to paths. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/codeql/codeql-config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index 55e7c9f..c4250ce 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -3,7 +3,12 @@ # by their own repos, so excluding them here avoids duplicate findings/build time. name: httrack-android CodeQL config +# NOTE: `paths` applies to ALL languages. Include the Java app sources (for the +# java-kotlin analysis) AND httrack-android's own JNI C (for c-cpp). The vendored +# engine/coffeecatch/libiconv are scanned by their own repos, so exclude them via +# paths-ignore to avoid duplicate findings. paths: + - app/src/main/java - app/src/main/jni/htslibjni.c - app/src/main/jni/htslibjni.h - app/src/main/jni/htslibjni-private.h From 01158e1539dd87589cea52038e1d94012d29b021 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Sun, 12 Jul 2026 19:14:02 +0200 Subject: [PATCH 5/5] ci: drop setup-ndk local-cache (restored NDK missing clang binaries) The first run downloaded a clean NDK and built; the next run restored an incomplete NDK from local-cache and failed with 'clang: No such file or directory'. Download the NDK fresh each run. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/android.yml | 3 ++- .github/workflows/codeql.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 83ccae8..63a119e 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -42,7 +42,8 @@ jobs: uses: nttld/setup-ndk@v1 with: ndk-version: ${{ env.NDK_VERSION }} - local-cache: true + # No local-cache: its restore can yield an NDK missing the clang + # binaries ("clang: No such file or directory"); download fresh each run. - name: Cache OpenSSL statics id: ssl-cache uses: actions/cache@v4 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ee21e13..34ad745 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -56,7 +56,7 @@ jobs: uses: nttld/setup-ndk@v1 with: ndk-version: ${{ env.NDK_VERSION }} - local-cache: true + # No local-cache: its restore can drop the NDK clang binaries. - name: Build native (c-cpp) if: matrix.language == 'c-cpp' env: