diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml new file mode 100644 index 0000000..c4250ce --- /dev/null +++ b/.github/codeql/codeql-config.yml @@ -0,0 +1,19 @@ +# Scope the c-cpp CodeQL analysis to httrack-android's OWN native code (the JNI +# glue). The vendored httrack engine, coucal, coffeecatch, and libiconv are scanned +# by their own repos, so excluding them here avoids duplicate findings/build time. +name: httrack-android CodeQL config + +# NOTE: `paths` applies to ALL languages. Include the Java app sources (for the +# java-kotlin analysis) AND httrack-android's own JNI C (for c-cpp). The vendored +# engine/coffeecatch/libiconv are scanned by their own repos, so exclude them via +# paths-ignore to avoid duplicate findings. +paths: + - app/src/main/java + - app/src/main/jni/htslibjni.c + - app/src/main/jni/htslibjni.h + - app/src/main/jni/htslibjni-private.h + +paths-ignore: + - app/src/main/jni/httrack + - app/src/main/jni/coffeecatch + - app/src/main/jni/libiconv-1.17 diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml new file mode 100644 index 0000000..63a119e --- /dev/null +++ b/.github/workflows/android.yml @@ -0,0 +1,78 @@ +# Android build + test for httrack-android. +# Style follows the httrack repo (least-privilege, concurrency cancel, submodules +# recursive). Phase A is self-contained: it sets up the NDK, cross-compiles OpenSSL +# (cached), vendors libiconv, and ndk-builds the native libs — no pre-published +# image needed. (The docker/ toolchain image + docker-image.yml exist for the later +# Gradle/assemble phase, which needs the full SDK.) +name: Android + +on: + push: + branches: [master] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: android-${{ github.ref }} + cancel-in-progress: true + +env: + NDK_VERSION: r26d + OPENSSL_VERSION: "3.0.15" + +jobs: + # Native-only: proves recursive submodules (incl. nested coucal), libiconv + # vendoring, OpenSSL 3.x, and 64-bit multi-ABI ndk-build. Validated locally for + # arm64-v8a + armeabi-v7a. x86_64 is added once the coffeecatch x86_64 fix lands + # (xroche/coffeecatch#46) — its legacy ucontext block #errors on x86_64. + native: + name: native (ndk-build) + runs-on: ubuntu-24.04 + env: + ABIS: arm64-v8a armeabi-v7a + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive # MUST be recursive: coucal is nested in httrack + - name: Set up NDK ${{ env.NDK_VERSION }} + id: ndk + uses: nttld/setup-ndk@v1 + with: + ndk-version: ${{ env.NDK_VERSION }} + # No local-cache: its restore can yield an NDK missing the clang + # binaries ("clang: No such file or directory"); download fresh each run. + - name: Cache OpenSSL statics + id: ssl-cache + uses: actions/cache@v4 + with: + path: /tmp/openssl-android + key: openssl-${{ env.OPENSSL_VERSION }}-ndk-${{ env.NDK_VERSION }}-${{ env.ABIS }} + - name: Build OpenSSL (cache miss) + if: steps.ssl-cache.outputs.cache-hit != 'true' + env: + ANDROID_NDK_ROOT: ${{ steps.ndk.outputs.ndk-path }} + run: OUT=/tmp/openssl-android OPENSSL_VERSION="${OPENSSL_VERSION}" bash docker/build-openssl.sh + - name: Vendor libiconv + run: tools/ci/vendor-libiconv.sh + - name: Stage OpenSSL statics into prebuild/ + run: OPENSSL_ANDROID_ROOT=/tmp/openssl-android tools/ci/fetch-openssl-statics.sh + - name: ndk-build + working-directory: app/src/main # contains jni/ — ndk-build auto-discovers Android.mk + Application.mk + run: ${{ steps.ndk.outputs.ndk-path }}/ndk-build APP_ABI="${ABIS}" APP_PLATFORM=android-24 -j"$(nproc)" + - uses: actions/upload-artifact@v4 + with: + name: native-libs + path: app/src/main/libs/**/*.so + if-no-files-found: warn + + # --- Phases below land with the app modernization (see the plan) --- + # + # assemble: ./gradlew assembleDebug assembleRelease lint — needs the AGP 8 / + # AndroidX migration (Phase 3), and can use the GHCR toolchain image. + # + # emulator-smoke: reactivecircus/android-emulator-runner (x86_64, api 21/29/34), + # installs the APK and runs an instrumented crawl against a local fixture + # server. Needs Phase 4 (storage) + the coffeecatch x86_64 fix. diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..34ad745 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,70 @@ +# CodeQL static analysis: the Java app AND the JNI/C glue. Mirrors the httrack +# repo's codeql.yml (security-extended). c-cpp is scoped to htslibjni*.c via +# .github/codeql/codeql-config.yml so the vendored engine isn't double-scanned. +# +# java-kotlin uses build-mode: none (the app doesn't build under modern Gradle +# until the AGP 8 migration in Phase 3 — no build needed to analyze the source). +# c-cpp builds the JNI glue via a self-contained NDK setup (no pre-published image). +name: CodeQL + +on: + push: + branches: [master] + pull_request: + schedule: + - cron: "31 4 * * 1" + +permissions: + contents: read + +concurrency: + group: codeql-${{ github.ref }} + cancel-in-progress: true + +env: + NDK_VERSION: r26d + OPENSSL_VERSION: "3.0.15" + +jobs: + analyze: + name: analyze (${{ matrix.language }}) + runs-on: ubuntu-24.04 + permissions: + contents: read + security-events: write + strategy: + fail-fast: false + matrix: + include: + - { language: java-kotlin, build-mode: none } + - { language: c-cpp, build-mode: manual } + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive + - uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + queries: security-extended + config-file: ./.github/codeql/codeql-config.yml + + # c-cpp only: CodeQL must observe the native compile (arm64-v8a is enough). + - name: Set up NDK ${{ env.NDK_VERSION }} + if: matrix.language == 'c-cpp' + id: ndk + uses: nttld/setup-ndk@v1 + with: + ndk-version: ${{ env.NDK_VERSION }} + # No local-cache: its restore can drop the NDK clang binaries. + - name: Build native (c-cpp) + if: matrix.language == 'c-cpp' + env: + ANDROID_NDK_ROOT: ${{ steps.ndk.outputs.ndk-path }} + run: | + OUT=/tmp/openssl-android ABIS=arm64-v8a OPENSSL_VERSION="${OPENSSL_VERSION}" bash docker/build-openssl.sh + tools/ci/vendor-libiconv.sh + OPENSSL_ANDROID_ROOT=/tmp/openssl-android ABIS=arm64-v8a tools/ci/fetch-openssl-statics.sh + ( cd app/src/main && "${ANDROID_NDK_ROOT}/ndk-build" APP_ABI=arm64-v8a APP_PLATFORM=android-24 -j"$(nproc)" ) + + - uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 0000000..93df4ff --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,48 @@ +# Build & publish the toolchain image (JDK17 + SDK + NDK + OpenSSL 3.x statics for +# all ABIs) to GHCR, so android.yml/codeql.yml pull a ready image instead of +# recompiling OpenSSL (~10-20 min) every run. First publish is private even for a +# public repo — set the package visibility to public once in repo Packages settings. +name: Toolchain image + +on: + push: + branches: [master] + paths: ['docker/**', '.github/workflows/docker-image.yml'] + schedule: + - cron: "23 5 * * 1" # weekly: pick up OpenSSL patch releases + workflow_dispatch: + +permissions: + contents: read + packages: write + +concurrency: + group: image-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-push: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/build-push-action@v6 + with: + context: docker + push: true + tags: | + ghcr.io/xroche/httrack-android-build:latest + ghcr.io/xroche/httrack-android-build:${{ github.sha }} + - name: Extract OpenSSL statics (for the container-less emulator job) + run: | + id=$(docker create ghcr.io/xroche/httrack-android-build:latest) + docker cp "$id":/opt/openssl-android ./openssl-android + docker rm "$id" + - uses: actions/upload-artifact@v4 + with: + name: openssl-android-statics + path: openssl-android diff --git a/.gitignore b/.gitignore index 3cc136d..20210a6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,9 @@ workspace.xml /app/.externalNativeBuild /app/build /app/src/main/jniLibs + +# Build inputs fetched/generated by tools/ci (not committed): +# libiconv vendored by tools/ci/vendor-libiconv.sh +# per-ABI OpenSSL statics from the toolchain image / fetch-openssl-statics.sh +/app/src/main/jni/libiconv-1.17/ +/app/src/main/prebuild/ diff --git a/app/src/main/java/com/httrack/android/jni/HTTrackLib.java b/app/src/main/java/com/httrack/android/jni/HTTrackLib.java index 7841b70..01ef9df 100755 --- a/app/src/main/java/com/httrack/android/jni/HTTrackLib.java +++ b/app/src/main/java/com/httrack/android/jni/HTTrackLib.java @@ -191,7 +191,7 @@ public static boolean loadLibraries() { try { /** Load all libraries. **/ - final String[] libraries = { "iconv", "httrack", "htsjava", "htslibjni" }; + final String[] libraries = { "iconv", "httrack", "htslibjni" }; for (final String lib : libraries) { Log.d(TAG, "Loading native library " + lib); System.loadLibrary(lib); diff --git a/app/src/main/jni/Android.mk b/app/src/main/jni/Android.mk index da43005..e845757 100755 --- a/app/src/main/jni/Android.mk +++ b/app/src/main/jni/Android.mk @@ -14,7 +14,15 @@ # LOCAL_PATH := $(call my-dir) -TARGET_ARCH := arm + +# The exidx-merge linker flag is armv7-only (ARM EXIDX sections exist only on +# 32-bit ARM); emitting it on arm64-v8a/x86_64 breaks the link. Gate it per-ABI. +# ndk-build re-includes this file once per ABI with TARGET_ARCH_ABI set. +ifeq ($(TARGET_ARCH_ABI),armeabi-v7a) + HTS_ARMV7_LDFLAGS := -Wl,--no-merge-exidx-entries +else + HTS_ARMV7_LDFLAGS := +endif # include $(CLEAR_VARS) @@ -27,17 +35,17 @@ LOCAL_CFLAGS := \ -DIN_LIBRARY \ -DLIBICONV_PLUG LOCAL_SRC_FILES := \ - libiconv-1.15/libcharset/lib/localcharset.c \ - libiconv-1.15/lib/iconv.c \ - libiconv-1.15/lib/relocatable.c + libiconv-1.17/libcharset/lib/localcharset.c \ + libiconv-1.17/lib/iconv.c \ + libiconv-1.17/lib/relocatable.c LOCAL_C_INCLUDES += \ $(LOCAL_PATH)/include/iconv \ - $(LOCAL_PATH)/libiconv-1.15/include \ - $(LOCAL_PATH)/libiconv-1.15/libcharset \ - $(LOCAL_PATH)/libiconv-1.15/lib \ - $(LOCAL_PATH)/libiconv-1.15/libcharset/include \ - $(LOCAL_PATH)/libiconv-1.15/srclib -LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)/libiconv-1.15/include + $(LOCAL_PATH)/libiconv-1.17/include \ + $(LOCAL_PATH)/libiconv-1.17/libcharset \ + $(LOCAL_PATH)/libiconv-1.17/lib \ + $(LOCAL_PATH)/libiconv-1.17/libcharset/include \ + $(LOCAL_PATH)/libiconv-1.17/srclib +LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)/libiconv-1.17/include include $(BUILD_SHARED_LIBRARY) # TODO FIXME: INVESTIGATE THIS @@ -52,12 +60,12 @@ include $(BUILD_SHARED_LIBRARY) include $(CLEAR_VARS) LOCAL_MODULE := libcrypto -LOCAL_SRC_FILES := ../prebuild/libcrypto.a +LOCAL_SRC_FILES := ../prebuild/$(TARGET_ARCH_ABI)/libcrypto.a include $(PREBUILT_STATIC_LIBRARY) include $(CLEAR_VARS) LOCAL_MODULE := libssl -LOCAL_SRC_FILES := ../prebuild/libssl.a +LOCAL_SRC_FILES := ../prebuild/$(TARGET_ARCH_ABI)/libssl.a include $(PREBUILT_STATIC_LIBRARY) include $(CLEAR_VARS) @@ -72,14 +80,18 @@ LOCAL_SRC_FILES := httrack/src/htscore.c httrack/src/htsparse.c \ httrack/src/htsmd5.c httrack/src/htszlib.c httrack/src/htswrap.c \ httrack/src/htsconcat.c httrack/src/htsmodules.c \ httrack/src/htscharset.c httrack/src/punycode.c \ - httrack/src/htsencoding.c httrack/src/md5.c httrack/src/minizip/ioapi.c \ + httrack/src/htsencoding.c httrack/src/md5.c \ + httrack/src/htssniff.c httrack/src/htsselftest.c \ + httrack/src/htscache_selftest.c httrack/src/htsdns_selftest.c \ + httrack/src/minizip/ioapi.c \ httrack/src/minizip/mztools.c httrack/src/minizip/unzip.c \ httrack/src/minizip/zip.c LOCAL_C_INCLUDES := $(LOCAL_PATH)/httrack/src \ $(LOCAL_PATH)/httrack/src/coucal \ + $(LOCAL_PATH)/../prebuild/$(TARGET_ARCH_ABI)/include \ $(LOCAL_PATH)/include LOCAL_LDLIBS := -ldl -lz -LOCAL_LDLIBS += -L$(LOCAL_PATH)/../prebuild +LOCAL_LDLIBS += -L$(LOCAL_PATH)/../prebuild/$(TARGET_ARCH_ABI) LOCAL_SHARED_LIBRARIES := libiconv LOCAL_STATIC_LIBRARIES := libssl libcrypto LOCAL_CFLAGS += -O3 -g3 -funwind-tables -fPIC -rdynamic \ @@ -96,35 +108,14 @@ LOCAL_CFLAGS += -O3 -g3 -funwind-tables -fPIC -rdynamic \ -D_REENTRANT -DPIC -DANDROID -D_ANDROID -DHAVE_CONFIG_H -DINET6 \ -DLIBHTTRACK_EXPORTS -DZLIB_CONST -DHTS_INTHASH_USES_MD5 -DLIBICONV_PLUG\ -DHTS_CRASH_TEST \ - -Wl,--no-merge-exidx-entries -Wl,-O1 + $(HTS_ARMV7_LDFLAGS) -Wl,-O1 LOCAL_CPPFLAGS += -pthread include $(BUILD_SHARED_LIBRARY) -include $(CLEAR_VARS) -LOCAL_MODULE := libhtsjava -LOCAL_SRC_FILES := httrack/src/htsjava.c -LOCAL_C_INCLUDES := $(LOCAL_PATH)/httrack/src \ - $(LOCAL_PATH)/httrack/src/coucal \ - $(LOCAL_PATH)/include -LOCAL_LDLIBS += -L$(LOCAL_PATH)/../prebuild -LOCAL_SHARED_LIBRARIES := libhttrack -LOCAL_CFLAGS += -O3 -g3 -funwind-tables -fPIC -rdynamic \ - -fstrict-aliasing -fvisibility=hidden \ - -Wall -Wformat -Wformat-security -Wmultichar -Wwrite-strings -Wcast-qual\ - -Wcast-align -Wstrict-prototypes -Wmissing-prototypes \ - -Wmissing-declarations -Wdeclaration-after-statement -Wpointer-arith \ - -Wsequence-point -Wnested-externs -Wparentheses -Winit-self \ - -Wunused-but-set-parameter -Waddress -Wuninitialized -Wformat=2 \ - -Wformat-nonliteral -Wmissing-parameter-type -Wold-style-definition \ - -Wignored-qualifiers -Wstrict-aliasing -Wno-sign-compare \ - -Wno-type-limits -Wno-missing-field-initializers -Wno-cast-align \ - -Wno-nested-externs \ - -D_REENTRANT -DPIC -DANDROID -D_ANDROID -DHAVE_CONFIG_H -DINET6 \ - -DLIBHTTRACK_EXPORTS -DZLIB_CONST -DHTS_INTHASH_USES_MD5 -DLIBICONV_PLUG\ - -DHTS_CRASH_TEST \ - -Wl,--no-merge-exidx-entries -Wl,-O1 -LOCAL_CPPFLAGS += -pthread -include $(BUILD_SHARED_LIBRARY) +# NOTE: the old `libhtsjava` module (httrack/src/htsjava.c) was dropped — the +# engine removed htsjava.c (the desktop Java-applet parser bridge) after 3.49-2, +# and nothing on Android used it. "htsjava" is likewise removed from +# HTTrackLib.loadLibraries(). include $(CLEAR_VARS) LOCAL_MODULE := htslibjni @@ -132,10 +123,11 @@ LOCAL_SRC_FILES := htslibjni.c coffeecatch/coffeecatch.c coffeecatch/coffeejni.c LOCAL_C_INCLUDES := $(LOCAL_PATH)/httrack/src \ $(LOCAL_PATH)/httrack/src/coucal \ $(LOCAL_PATH)/coffeecatch \ + $(LOCAL_PATH)/../prebuild/$(TARGET_ARCH_ABI)/include \ $(LOCAL_PATH)/include LOCAL_SHARED_LIBRARIES := libhttrack LOCAL_LDLIBS := -llog LOCAL_CFLAGS := -O3 -g -funwind-tables \ - -Wl,--no-merge-exidx-entries -Wl,-O1 \ + $(HTS_ARMV7_LDFLAGS) -Wl,-O1 \ -W -Wall -Wextra -Werror -Wno-unused-parameter include $(BUILD_SHARED_LIBRARY) diff --git a/app/src/main/jni/Application.mk b/app/src/main/jni/Application.mk index 6d00145..d9bef21 100755 --- a/app/src/main/jni/Application.mk +++ b/app/src/main/jni/Application.mk @@ -1 +1,10 @@ -APP_ABI := armeabi-v7a +# 64-bit multi-ABI: arm64-v8a is the primary target, armeabi-v7a kept for old +# devices, x86_64 for emulators/CI. 64-bit ABIs require API 21+. +# +# APP_PLATFORM is android-24, not 21: httrack's bundled minizip (ioapi.c) calls +# fseeko/ftello for large-file (>2GB) zip-cache seeks, and bionic only introduces +# them at API 24. Keeping the min at 21 would require an upstream minizip guard +# (fall back to fseek/ftell below 24, capping the cache at 2GB). See the +# android-modernization plan — this is a min-API decision for Xavier. +APP_ABI := arm64-v8a armeabi-v7a x86_64 +APP_PLATFORM := android-24 diff --git a/app/src/main/jni/httrack b/app/src/main/jni/httrack index e8bdccc..627ae2b 160000 --- a/app/src/main/jni/httrack +++ b/app/src/main/jni/httrack @@ -1 +1 @@ -Subproject commit e8bdccc443f80ea5d0ce6dff5916625bdd81504e +Subproject commit 627ae2b04331ca9df54100c336558d857dcc8f0e diff --git a/app/src/main/jni/include/config.h b/app/src/main/jni/include/config.h index 87e8c9c..f950fef 100755 --- a/app/src/main/jni/include/config.h +++ b/app/src/main/jni/include/config.h @@ -105,11 +105,12 @@ /* Check for setuid */ #define SETUID 1 -/* The size of `long', as computed by sizeof. */ -#define SIZEOF_LONG 4 +/* The size of `long'/`long long', as computed by sizeof. This config.h is + hand-maintained and shared across ABIs, so DON'T hardcode 4 here: sizeof(long) + is 8 on arm64-v8a and x86_64. Derive it from the compiler instead. */ +#define SIZEOF_LONG (sizeof(long)) -/* The size of `long long', as computed by sizeof. */ -#define SIZEOF_LONG_LONG 8 +#define SIZEOF_LONG_LONG (sizeof(long long)) /* Define to 1 if you have the ANSI C header files. */ #define STDC_HEADERS 1 diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..62ba0c4 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,63 @@ +# Reproducible build environment for the modernized HTTrack Android app. +# +# - JDK 17 (required by AGP 8.x) +# - Android cmdline-tools + platform 35 + build-tools 35 +# - NDK r26d (Clang, unified toolchain — replaces the r14/GCC hack) +# - OpenSSL 3.x static libs cross-compiled for arm64-v8a/armeabi-v7a/x86_64 +# +# Build: docker build -t httrack-android-build docker/ +# Use: docker run --rm -v "$PWD":/src -w /src httrack-android-build \ +# ./gradlew assembleRelease +# +# The OpenSSL libs are baked into the image under /opt/openssl-android//. +# Point Android.mk's prebuilt paths at them (or copy into app/src/main/prebuild). +FROM eclipse-temurin:17-jdk-jammy + +ARG ANDROID_CMDLINE_VERSION=11076708 +ARG ANDROID_PLATFORM=android-35 +ARG ANDROID_BUILD_TOOLS=35.0.0 +ARG NDK_VERSION=26.3.11579264 +ARG OPENSSL_VERSION=3.0.15 +ARG OPENSSL_SHA256=23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533 + +ENV ANDROID_SDK_ROOT=/opt/android-sdk \ + ANDROID_HOME=/opt/android-sdk \ + DEBIAN_FRONTEND=noninteractive + +# NOTE: use wget, not curl, for downloads. On this host's Docker/kernel combo +# curl's threaded DNS resolver fails ("getaddrinfo() thread failed to start") +# even under seccomp=unconfined, while wget's synchronous resolver works. +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget unzip git make perl file ca-certificates build-essential && \ + rm -rf /var/lib/apt/lists/* +# build-essential: needed by tools/ci/vendor-libiconv.sh (host ./configure) and +# by OpenSSL's build. wget (not curl): curl's threaded resolver fails on some +# Docker/kernel combos ("getaddrinfo() thread failed to start"). + +# --- Android SDK command-line tools --- +RUN mkdir -p "$ANDROID_SDK_ROOT/cmdline-tools" && cd /tmp && \ + wget -q "https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_CMDLINE_VERSION}_latest.zip" && \ + unzip -q "commandlinetools-linux-${ANDROID_CMDLINE_VERSION}_latest.zip" && \ + mv cmdline-tools "$ANDROID_SDK_ROOT/cmdline-tools/latest" && \ + rm "commandlinetools-linux-${ANDROID_CMDLINE_VERSION}_latest.zip" +ENV PATH="$ANDROID_SDK_ROOT/cmdline-tools/latest/bin:$ANDROID_SDK_ROOT/platform-tools:$PATH" + +# --- SDK packages + NDK --- +RUN yes | sdkmanager --licenses >/dev/null && \ + sdkmanager --install \ + "platform-tools" \ + "platforms;${ANDROID_PLATFORM}" \ + "build-tools;${ANDROID_BUILD_TOOLS}" \ + "ndk;${NDK_VERSION}" >/dev/null +ENV ANDROID_NDK_ROOT="$ANDROID_SDK_ROOT/ndk/${NDK_VERSION}" + +# --- OpenSSL 3.x static libs for all ABIs --- +COPY build-openssl.sh /usr/local/bin/build-openssl.sh +RUN chmod +x /usr/local/bin/build-openssl.sh && \ + OPENSSL_VERSION="${OPENSSL_VERSION}" OPENSSL_SHA256="${OPENSSL_SHA256}" \ + OUT=/opt/openssl-android MIN_API=21 \ + /usr/local/bin/build-openssl.sh && \ + rm -rf /tmp/* +ENV OPENSSL_ANDROID_ROOT=/opt/openssl-android + +WORKDIR /src diff --git a/docker/build-openssl.sh b/docker/build-openssl.sh new file mode 100755 index 0000000..d982b79 --- /dev/null +++ b/docker/build-openssl.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# Cross-compile OpenSSL 3.x static libs for Android, all target ABIs, using the +# NDK's own Clang toolchain. Replaces the dead 2017 setenv-android.sh + OpenSSL +# 1.0.2k hack (see ../openssl-ndk-analysis.md). +# +# Output layout (consumed by app/src/main/jni/Android.mk): +# $OUT//lib/libssl.a +# $OUT//lib/libcrypto.a +# $OUT//include/openssl/*.h +# +# Requires: ANDROID_NDK_ROOT set, wget, perl, make. Run inside the Docker image. +# (wget, not curl: curl's threaded resolver fails on this host's Docker/kernel.) +set -euo pipefail + +OPENSSL_VERSION="${OPENSSL_VERSION:-3.0.15}" +# Pin the hash — verify against https://www.openssl.org/source/ before bumping. +OPENSSL_SHA256="${OPENSSL_SHA256:-23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533}" +MIN_API="${MIN_API:-21}" +OUT="${OUT:-/opt/openssl-android}" +ABIS="${ABIS:-arm64-v8a armeabi-v7a x86_64}" + +: "${ANDROID_NDK_ROOT:?set ANDROID_NDK_ROOT}" +HOST_TAG="linux-x86_64" +export PATH="$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/$HOST_TAG/bin:$PATH" + +abi_to_target() { + case "$1" in + arm64-v8a) echo android-arm64 ;; + armeabi-v7a) echo android-arm ;; + x86_64) echo android-x86_64 ;; + x86) echo android-x86 ;; + *) echo "unknown ABI: $1" >&2; exit 1 ;; + esac +} + +work="$(mktemp -d)" +cd "$work" +wget -q "https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/openssl-${OPENSSL_VERSION}.tar.gz" +echo "${OPENSSL_SHA256} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c - +tar xf "openssl-${OPENSSL_VERSION}.tar.gz" +src="$work/openssl-${OPENSSL_VERSION}" + +for abi in $ABIS; do + target="$(abi_to_target "$abi")" + echo "=== Building OpenSSL ${OPENSSL_VERSION} for ${abi} (${target}) ===" + builddir="$work/build-$abi" + cp -a "$src" "$builddir" + ( cd "$builddir" + ./Configure "$target" "-D__ANDROID_API__=${MIN_API}" \ + no-shared no-tests no-ui-console no-engine no-comp no-dso no-legacy \ + --prefix="$OUT/$abi" --libdir=lib + make -j"$(nproc)" build_libs + make install_dev + ) +done + +echo "=== OpenSSL static libs installed under $OUT// ===" +find "$OUT" -name 'lib*.a' -printf '%p\n' diff --git a/tools/ci/fetch-openssl-statics.sh b/tools/ci/fetch-openssl-statics.sh new file mode 100755 index 0000000..25358f9 --- /dev/null +++ b/tools/ci/fetch-openssl-statics.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# Stage per-ABI OpenSSL static libs + headers into app/src/main/prebuild//, +# the layout Android.mk expects (../prebuild/$(TARGET_ARCH_ABI)/lib{ssl,crypto}.a +# and .../include/openssl). +# +# Source of the statics, in priority order: +# 1. $OPENSSL_ANDROID_ROOT (set in the toolchain Docker image → /opt/openssl-android) +# 2. build them here from a standalone NDK (needs $ANDROID_NDK_ROOT), via +# docker/build-openssl.sh — used by the container-less emulator job. +set -euo pipefail + +JNI_DIR="$(cd "$(dirname "$0")/../../app/src/main/jni" && pwd)" +PREBUILD="$JNI_DIR/../prebuild" +ABIS="${ABIS:-arm64-v8a armeabi-v7a x86_64}" +SRC="${OPENSSL_ANDROID_ROOT:-}" + +if [ -z "$SRC" ] || [ ! -d "$SRC" ]; then + echo "OPENSSL_ANDROID_ROOT not set/found — building OpenSSL from NDK ..." + : "${ANDROID_NDK_ROOT:?set ANDROID_NDK_ROOT or OPENSSL_ANDROID_ROOT}" + SRC="${OUT:-/tmp/openssl-android}" + OUT="$SRC" ABIS="$ABIS" bash "$(dirname "$0")/../../docker/build-openssl.sh" +fi + +for abi in $ABIS; do + mkdir -p "$PREBUILD/$abi/include" + cp "$SRC/$abi/lib/libssl.a" "$SRC/$abi/lib/libcrypto.a" "$PREBUILD/$abi/" + cp -r "$SRC/$abi/include/openssl" "$PREBUILD/$abi/include/" +done +echo "staged OpenSSL statics for: $ABIS" diff --git a/tools/ci/vendor-libiconv.sh b/tools/ci/vendor-libiconv.sh new file mode 100755 index 0000000..6b502ee --- /dev/null +++ b/tools/ci/vendor-libiconv.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# Vendor GNU libiconv into app/src/main/jni/libiconv-1.17/ and generate the +# headers the ndk-build needs (iconv.h from iconv.h.in, config.h). bionic has no +# iconv, so httrack bundles it (built with -DLIBICONV_PLUG). Only 3 source files +# are compiled by Android.mk; iconv.c is a mega-TU that #includes every converter, +# so it needs configure-generated headers + the tarball's pre-generated aliases.h. +# +# Idempotent: re-running with the tree already present is a no-op. Needs a host C +# compiler + make for ./configure (header generation only; the actual cross-build +# is done by ndk-build). +set -euo pipefail + +VERSION="${LIBICONV_VERSION:-1.17}" +SHA256="${LIBICONV_SHA256:-8f74213b56238c85a50a5329f77e06198771e70dd9a739779f4c02f65d971313}" +JNI_DIR="$(cd "$(dirname "$0")/../../app/src/main/jni" && pwd)" +DEST="$JNI_DIR/libiconv-${VERSION}" + +if [ -f "$DEST/include/iconv.h" ] && [ -f "$DEST/config.h" ]; then + echo "libiconv ${VERSION} already vendored + configured at $DEST" + exit 0 +fi + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +cd "$work" +echo "=== fetching libiconv ${VERSION} ===" +wget -q "https://ftp.gnu.org/gnu/libiconv/libiconv-${VERSION}.tar.gz" +echo "${SHA256} libiconv-${VERSION}.tar.gz" | sha256sum -c - +tar xf "libiconv-${VERSION}.tar.gz" + +echo "=== configure (host) to generate iconv.h + config.h ===" +( cd "libiconv-${VERSION}" && ./configure --enable-static --disable-shared >/dev/null ) + +rm -rf "$DEST" +mkdir -p "$(dirname "$DEST")" +mv "libiconv-${VERSION}" "$DEST" + +# The host ./configure detects HAVE_LANGINFO_CODESET (true on Linux), but bionic +# only introduces nl_langinfo() at API 26. localcharset.c is built without +# -DHAVE_CONFIG_H and reads whichever config.h is on the include path, so neutralize +# the define in ALL generated config.h. Android is always UTF-8, so the non- +# nl_langinfo path is correct. (Cross-config skew; a proper NDK cross-configure +# would set this automatically.) +for f in "$DEST/config.h" "$DEST/libcharset/config.h" "$DEST/lib/config.h"; do + [ -f "$f" ] && sed -i \ + 's|^#define HAVE_LANGINFO_CODESET 1|/* #undef HAVE_LANGINFO_CODESET (bionic nl_langinfo is API 26+) */|' "$f" +done + +echo "=== vendored at $DEST ===" +ls "$DEST/include/iconv.h" "$DEST/config.h" "$DEST/lib/aliases.h" 2>&1