PostHog · dustinbyrne · Sep 8, 2026 · Sep 8, 2026
@@ -2,8 +2,6 @@ name: SDK Compliance Tests
 
 permissions:
   contents: read
-  packages: read
-  pull-requests: write
 
 on:
   pull_request:
@@ -12,10 +10,63 @@ on:
       - main
 
 jobs:
-  compliance:
-    name: PostHog SDK compliance tests
-    uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@03d972e49be84402c491324320b0a0f38c2ddc53 # main @ 2026-07-27
-    with:
-      adapter-dockerfile: "sdk_compliance_adapter/Dockerfile"
-      adapter-context: "."
-      test-harness-version: "0.9.0"
+  macos-flutter:
+    name: Flutter MethodChannel / macOS
+    runs-on: macos-26
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: axi92/flutter-action@36d2c2625bac6ea011cd7808d2a01bd8a7e5c766
+        with:
+          flutter-version: '3.44.0'
+          channel: stable
+          cache: true
+      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          version: '0.12.10'
+          python-version: '3.12.12'
+          activate-environment: true
+          venv-path: ${{ runner.temp }}/harness-venv
+          enable-cache: false
+        env:
+          UV_PYTHON_INSTALL_DIR: ${{ runner.temp }}/managed-python
+      # Native runners do not require a Docker engine. This is the unchanged
+      # release source corresponding to ghcr.io/posthog/sdk-test-harness:1.0.0.
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: PostHog/posthog-sdk-test-harness
+          ref: 6d19abb9c81e2262dacbe340e7dddda9c871c178 # 1.0.0
+          path: harness
+      - name: Install harness
+        run: |
+          uv pip install --python "$VIRTUAL_ENV/bin/python" -e ./harness
+          mkdir -p "$RUNNER_TEMP/flutter-report"
+          python -c 'import sys; assert sys.version_info[:3] == (3, 12, 12); print(sys.version); print(sys.executable)' \
+            > "$RUNNER_TEMP/flutter-report/python-version.txt"
+      - name: Build real Flutter application and test observer
+        run: |
+          set -o pipefail
+          mkdir -p "$RUNNER_TEMP/flutter-report"
+          bash sdk_compliance_adapter/build_macos.sh "$RUNNER_TEMP/flutter-compliance" \
+            2>&1 | tee "$RUNNER_TEMP/flutter-report/build.log"
+      - name: Run advisory compliance assertions
+        continue-on-error: true
+        run: |
+          bash sdk_compliance_adapter/run_macos.sh \
+            "$RUNNER_TEMP/flutter-compliance" "$GITHUB_WORKSPACE/harness" \
+            "$RUNNER_TEMP/flutter-report"
+      - name: Require complete nonempty inventory
+        if: always()
+        run: |
+          python sdk_compliance_adapter/check_report.py "$RUNNER_TEMP/flutter-report/report.json"
+      - name: Upload macOS Flutter results and resolved delegates
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: sdk-compliance-macos-flutter-method-channel
+          path: |
+            ${{ runner.temp }}/flutter-report/*.json
+            ${{ runner.temp }}/flutter-report/*.log
+            ${{ runner.temp }}/flutter-report/*.txt
+            ${{ runner.temp }}/flutter-report/*.lock
+          if-no-files-found: error
@@ -1,11 +1,108 @@
-# PostHog Flutter SDK compliance adapter
+# Flutter macOS compliance profile
 
-HTTP adapter used by the PostHog SDK compliance harness.
+This adapter runs a **real macOS Flutter application**. Calls go through the
+public `Posthog` Dart facade, the shipped MethodChannel implementation and the
+registered Apple plugin. The resolved PostHog CocoaPod owns event construction,
+UUIDs, timestamps, batching, compression, retries, flag parsing and flag-called
+events. This profile does not certify Android, iOS devices/simulators or web.
 
-The adapter exposes `/health`, `/init`, `/capture`, `/flush`, `/state`, and `/reset` on port `8080`.
+## Run
 
-Run locally:
+Requires macOS, Xcode, CocoaPods, Flutter **3.44.0** (Dart 3.12), and an unchanged
+[SDK harness](https://github.com/PostHog/posthog-sdk-test-harness) checkout with
+its Python dependencies installed. CI pins the native harness source to release
+**1.0.0**, `6d19abb9c81e2262dacbe340e7dddda9c871c178`, rather than requiring Docker
+on the macOS runner.
+
+From the repository root, using new, absolute output directories:
 
 ```sh
-docker compose -f sdk_compliance_adapter/docker-compose.yml up --build --abort-on-container-exit --exit-code-from test-harness
+bash sdk_compliance_adapter/build_macos.sh /tmp/flutter-compliance-build
+bash sdk_compliance_adapter/run_macos.sh \
+  /tmp/flutter-compliance-build /path/to/posthog-sdk-test-harness \
+  /tmp/flutter-compliance-report
+python3 sdk_compliance_adapter/check_report.py /tmp/flutter-compliance-report/report.json
 ```
+
+`PYTHON` can select the harness virtualenv interpreter. `PORT`, `MOCK_PORT` and
+`PROXY_PORT` default to 18310, 19310 and 19311. `APP_PORT` defaults to 18311.
+Observer unit tests use 19312/19313; the native regression also uses 18312/18313.
+The runner checks for occupied service ports and cleans up its owned processes.
+
+The build script creates an isolated workspace containing unmodified SDK sources
+and manifests, the adapter and a generated macOS runner. It does not build the
+example or relax SDK dependency constraints. CocoaPods metadata is isolated too.
+The generated app disables automatic initialization and the app sandbox; it is
+only a local test controller, not an app distribution template.
+
+Reports include Flutter/Dart versions, wrapper version, **resolved** Apple
+version, `Podfile.lock`, Dart lockfile, harness revision, health, and logs. The
+initial validated combination is Flutter 3.44.0 / wrapper 5.39.0 / Apple 3.71.6;
+subsequent builds resolve within the SDK's unchanged native dependency range.
+
+## Mapping and observation limits
+
+- The Python HTTP controller starts the real Flutter binary with a fresh owned
+  `CFFIXED_USER_HOME`, `HOME` and `TMPDIR` for every `/init`. `/reset` terminates
+  that app, waits for exit, and removes only its whole temporary environment.
+  Native SDK reset intentionally preserves queued events, so it cannot isolate
+  harness cases. This profile tests process isolation, not native reset semantics.
+- `/init` forwards host, `flush_at` and `flush_interval_ms`. Application lifecycle
+  capture and startup flag preload are explicitly disabled. Flag-called events
+  retain their SDK default. Native configuration serialization, including
+  whole-second interval precision, is unchanged. The adapter's default interval
+  is one second, the smallest positive interval supported by the Apple bridge.
+- Capture identity changes use public `identify`; flags use public identity,
+  person/group setters, awaited `reloadFeatureFlags` and cached `getFeatureFlag`.
+  `force_remote: false` omits the explicit reload, not any setter-triggered work.
+  **Identify/group events and automatic reloads are preserved.** This can affect
+  event counts and consume mock responses before the requested capture/reload.
+  Reload-per-action is not a claim about ordinary cached getter network behavior.
+- There is no public Flutter timestamp override, returned capture UUID, analytics
+  retry-budget/compression switch, queue state or per-getter GeoIP/singleton scope
+  control. Timestamp requests return HTTP 501; capture UUID and flush counts are
+  `null`; `/state` returns HTTP 501. Unsupported configuration is not synthesized.
+- A loopback proxy forwards each request and response once, preserving body bytes,
+  headers (except transport hop headers), response codes and retry headers. It
+  never retries. Per-initialization URL prefixes isolate late requests from closed
+  SDK instances; the prefix is removed when forwarding to the mock host.
+- `/flush` calls public flush **once** and waits up to 30 seconds for submitted
+  capture names to be observed under SDK-generated UUIDs with terminal HTTP
+  responses, no observed outstanding batches, and one second of network quiet.
+  Unobserved captures, unresolved retry batches and observation errors fail the
+  action explicitly. With no submitted captures, the action returns HTTP 501
+  after invoking public flush: network silence cannot prove an empty queue or
+  automatic-event completion. This is a bounded capture-wire observation,
+  **not a native queue drain guarantee**. Automatic SDK events and permanently
+  exhausted retries have no public completion callback. No timer or flush loop
+  drives SDK retries. `/observations` exposes passive request records, not SDK
+  queue state.
+
+## Selected coverage
+
+`expected-tests.json` lists all **47** selected contract 1.2 definitions:
+30 server-wire capture and 17 flags cases, with `capture_v0` and `encoding_gzip`.
+No individual failures are filtered. CI keeps assertion failures advisory but
+fails on missing, duplicate, unexpected or zero-case reports.
+
+The supported first-pass targets are 29 capture cases (excluding the absent
+explicit timestamp override), flag wire fields/path, groups, returned values,
+502/504 retries, flag-called events, and explicit reload-per-action. Results can
+still fail because of real SDK policy, identity side effects, mock response
+routing, or bounded flush observation; a passing status test is not sufficient
+proof of capture retry behavior if another SDK request consumed its mock status.
+
+Deferred assertions remain visible in the full report: timestamp override,
+compound person/default-empty-group payloads, GeoIP, singleton key scope and
+native-default preload lifecycle. Apple omits fields some server-oriented flags
+assertions require. Web flags/gzip applicability, other platforms, V1/AI,
+alternate codecs and exact native queue-drain/state contracts remain uncovered.
+
+The resolved Apple 3.71.6 parser requires both `featureFlags` and
+`featureFlagPayloads` maps; legacy-only harness fixtures can therefore return
+`null` through the real getter. The harness fixtures are unchanged. A separate
+native regression uses a compatible mock response to verify public reload/getter
+502/504 retries and SDK-generated flag-called events. It also verifies reset
+isolation with an unsent event and native capture 503-to-200 retry identity. These
+focused checks run against the built app as a build gate, separately from the
+advisory 47-case harness inventory.
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euo pipefail
+ROOT=$(cd "$(dirname "$0")/.." && pwd)
+WORKSPACE=${1:?Usage: build_macos.sh NEW_WORKSPACE_DIRECTORY}
+python3 "$ROOT/sdk_compliance_adapter/prepare_macos.py" "$WORKSPACE"
+WORKSPACE=$(cd "$WORKSPACE" && pwd)
+export CP_HOME_DIR="$WORKSPACE/.cocoapods"
+export COCOAPODS_DISABLE_STATS=true
+cd "$WORKSPACE"
+flutter --version > toolchain.txt
+flutter pub get
+# Refresh only this build's CocoaPods metadata, not the developer's shared repo.
+pod repo add-cdn trunk https://cdn.cocoapods.org/
+pod repo update
+cd sdk_compliance_adapter
+flutter test test/wire_observer_test.dart
+dart analyze lib test
+cd ../runner
+flutter build macos --release --config-only
+SDK_VERSION=$(sed -n 's/^version: //p' ../posthog_flutter/pubspec.yaml)
+DELEGATE_VERSION=$(sed -n 's/^  - PostHog (\([^)]*\)).*/\1/p' macos/Podfile.lock)
+test -n "$SDK_VERSION"
+test -n "$DELEGATE_VERSION"
+printf 'Flutter wrapper: %s\nApple delegate (CocoaPods): %s\n' "$SDK_VERSION" "$DELEGATE_VERSION" > ../delegate-versions.txt
+flutter build macos --release \
+  --dart-define="SDK_VERSION=$SDK_VERSION" \
+  --dart-define="DELEGATE_VERSION=$DELEGATE_VERSION"
+FLUTTER_COMPLIANCE_BINARY="$PWD/build/macos/Build/Products/Release/flutter_compliance.app/Contents/MacOS/flutter_compliance" \
+  TEST_OUTPUT="$WORKSPACE" python3 "$ROOT/sdk_compliance_adapter/test_native_runtime.py" -v
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+"""Fail on missing, duplicate or unexpected cases, independently of compliance."""
+import json
+from pathlib import Path
+import sys
+
+expected = json.loads((Path(__file__).parent / 'expected-tests.json').read_text())
+report = json.loads(Path(sys.argv[1]).read_text())
+actual = [f"{suite['name']}.{case['name']}"
+          for suite in report['suites'] for case in suite['tests']]
+assert expected and sorted(actual) == sorted(expected), (
+    f'Inventory mismatch: missing={set(expected) - set(actual)}, '
+    f'extra={set(actual) - set(expected)}, count={len(actual)}')
+assert report['summary']['total'] == len(expected)
+print(f"Verified {len(actual)} macOS Flutter cases: {report['summary']}")
+for suite in report['suites']:
+    for case in suite['tests']:
+        if not case['passed']:
+            print(f"FAIL {suite['name']}.{case['name']}: {case['message']}")
@@ -0,0 +1,49 @@
+[
+  "capture.format_validation.event_has_required_fields",
+  "capture.format_validation.event_has_uuid",
+  "capture.format_validation.event_has_lib_properties",
+  "capture.format_validation.distinct_id_is_string",
+  "capture.format_validation.token_is_present",
+  "capture.format_validation.custom_properties_preserved",
+  "capture.format_validation.event_has_timestamp",
+  "capture.format_validation.non_utc_event_timestamp_is_converted_to_utc",
+  "capture.retry_behavior.retries_on_503",
+  "capture.retry_behavior.does_not_retry_on_400",
+  "capture.retry_behavior.does_not_retry_on_401",
+  "capture.retry_behavior.respects_retry_after_header",
+  "capture.retry_behavior.implements_backoff",
+  "capture.retry_behavior.retries_on_500",
+  "capture.retry_behavior.retries_on_502",
+  "capture.retry_behavior.retries_on_504",
+  "capture.retry_behavior.max_retries_respected",
+  "capture.deduplication.generates_unique_uuids",
+  "capture.deduplication.preserves_uuid_on_retry",
+  "capture.deduplication.preserves_uuid_and_timestamp_on_retry",
+  "capture.deduplication.preserves_uuid_and_timestamp_on_batch_retry",
+  "capture.deduplication.no_duplicate_events_in_batch",
+  "capture.deduplication.different_events_have_different_uuids",
+  "capture.compression.sends_gzip_when_enabled",
+  "capture.batch_format.uses_proper_batch_structure",
+  "capture.batch_format.flush_with_no_events_sends_nothing",
+  "capture.batch_format.multiple_events_batched_together",
+  "capture.error_handling.does_not_retry_on_403",
+  "capture.error_handling.does_not_retry_on_413",
+  "capture.error_handling.retries_on_408",
+  "feature_flags.request_payload.request_with_person_properties_device_id",
+  "feature_flags.request_payload.flags_request_uses_v2_query_param",
+  "feature_flags.request_payload.flags_request_hits_flags_path_not_decide",
+  "feature_flags.request_payload.flags_request_omits_authorization_header",
+  "feature_flags.request_payload.token_in_flags_body_matches_init",
+  "feature_flags.request_payload.groups_round_trip",
+  "feature_flags.request_payload.groups_default_to_empty_object",
+  "feature_flags.request_payload.disable_geoip_false_propagates_as_geoip_disable_false",
+  "feature_flags.request_payload.disable_geoip_omitted_defaults_to_false",
+  "feature_flags.request_payload.flag_keys_to_evaluate_contains_only_requested_key",
+  "feature_flags.request_lifecycle.no_flags_request_on_init_alone",
+  "feature_flags.request_lifecycle.no_flags_request_on_normal_capture",
+  "feature_flags.request_lifecycle.two_flag_calls_produce_two_remote_requests",
+  "feature_flags.request_lifecycle.mock_response_value_is_returned_to_caller",
+  "feature_flags.retry_behavior.retries_flags_on_502",
+  "feature_flags.retry_behavior.retries_flags_on_504",
+  "feature_flags.side_effect_events.get_feature_flag_captures_feature_flag_called_event"
+]