diff --git a/.github/workflows/build-awscrt.yml b/.github/workflows/build-awscrt.yml index 22bb0eed8c..7b0208c8a0 100644 --- a/.github/workflows/build-awscrt.yml +++ b/.github/workflows/build-awscrt.yml @@ -16,11 +16,13 @@ on: paths: - '.github/workflows/build-awscrt.yml' - 'docs/packages/awscrt.yaml' + - 'patches/awscrt/**' push: branches: [main] paths: - '.github/workflows/build-awscrt.yml' - 'docs/packages/awscrt.yaml' + - 'patches/awscrt/**' concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -55,6 +57,10 @@ jobs: # Each abi3 wheel is built on the oldest interpreter its tag claims and only # re-tested on the next one, so cp311 is here despite our cp312 floor: awscrt # defines PY_SSIZE_T_CLEAN and parses "s#"/"z#" throughout. + # wheel_tag is a real matrix dimension (not include-only) so it fans out + # against every pending version instead of collapsing to one combination + # per include entry (CLAUDE.md gotcha 402). + wheel_tag: [cp311-abi3, cp313-abi3, cp314-cp314t] include: - wheel_tag: cp311-abi3 builds: cp311-manylinux_riscv64 cp312-manylinux_riscv64 @@ -82,6 +88,20 @@ jobs: python3 continuous-delivery/update-version.py grep -q "__version__ = '${AWSCRT_VERSION}'" awscrt/__init__.py + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + # Only some versions need patching, so a missing directory is not an error. + - name: Apply patches + run: | + patches="$PWD/python-wheels/patches/awscrt/${AWSCRT_VERSION}" + if [ -d "$patches" ]; then + git apply -v "$patches"/*.patch + fi + - name: Build wheels uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 with: diff --git a/docs/packages/awscrt.yaml b/docs/packages/awscrt.yaml index 53ebf445cd..7baf2d3c62 100644 --- a/docs/packages/awscrt.yaml +++ b/docs/packages/awscrt.yaml @@ -16,3 +16,4 @@ versions: requires-python: '>=3.8' - version: 0.36.3 - version: 0.36.4 +- version: 0.37.0 diff --git a/patches/awscrt/0.36.3/0001-test-let-the-stream-be-freed-before-stopping-the-local-server.patch b/patches/awscrt/0.36.3/0001-test-let-the-stream-be-freed-before-stopping-the-local-server.patch new file mode 100644 index 0000000000..647b8301fc --- /dev/null +++ b/patches/awscrt/0.36.3/0001-test-let-the-stream-be-freed-before-stopping-the-local-server.patch @@ -0,0 +1,118 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 25 Sep 2026 00:00:00 +0000 +Subject: [PATCH] test: let the stream be freed before stopping the local + server on free-threaded builds + +TestClient and TestAsyncClient's _test_stream_lives_until_complete drop +every local reference to the stream and the connection, wait for the +response, then call _stop_server(). That calls HTTPServer.shutdown(), and +the single-threaded HTTPServer is still inside handle() for the HTTP/1.1 +keep-alive connection. So shutdown() only returns once the client socket +closes, which happens only when the Python stream (which holds the +connection) is deallocated. + +With the GIL, the stream's last reference is dropped by the DECREFs in +s_on_stream_complete on the native event-loop thread, and the stream, +connection and socket are freed right there. On a free-threaded build, an +object's owner thread is the one that created it (here the test's main +thread). A DECREF from any other thread that takes the shared refcount +negative only queues the object for its owner to merge. The owner merges +the next time it runs bytecode, i.e. when the eval breaker fires. If the main +thread is already parked in shutdown()'s Event.wait() by then, nothing ever +merges the refcount. The stream never dies, the socket never closes and +the job hangs until the CI timeout. Whether it hangs depends on whether +the main thread gets from the completion wake-up into shutdown() before +the native thread's final DECREF. + +That is what happened on this repo's riscv64 runners. 0.36.4 cp314t hung in +test_http_client's test_stream_lives_until_complete_http, and on a +later run 0.36.3 cp314t hung in test_aiohttp_client's. Each version +passes on other runs, and test/ is byte-identical across 0.36.3, 0.36.4 +and 0.37.0. It is not specific to riscv64: with upstream's own +awscrt 0.36.3 cp314-cp314t x86_64 wheel from PyPI, a reproducer that +reaches shutdown() before the response arrives hangs 3/3 on 3.14.7t, +while cp314 returns in 0.5s 3/3. + +Keep a weakref to the stream and, after the completion check, run +bytecode until it is freed (bounded by the test's own timeout) before +stopping the server. Nothing else changes: the tests still check that the +stream completes after every local reference is gone, and now also +wait for it to be released instead of relying on the GIL's immediate +deallocation. + +Upstream-Status: To upstream [latent free-threading deadlock in upstream's own test harness, reproduced on x86_64 with upstream's PyPI wheel; not yet sent, since that needs a contributor-signed upstream PR, which this repo's automation does not open] + +Signed-off-by: Ludovic Henry +--- +diff --git a/test/test_aiohttp_client.py b/test/test_aiohttp_client.py +index 8d24c39..57facfb 100644 +--- a/test/test_aiohttp_client.py ++++ b/test/test_aiohttp_client.py +@@ -3,6 +3,7 @@ + + import time + import socket ++import weakref + import sys + import asyncio + import unittest +@@ -235,6 +236,8 @@ class TestAsyncClient(AsyncLocalServerTestBase): + # Schedule task to collect response but don't await it yet + collect_task = asyncio.create_task(response.collect_response(stream)) + ++ stream_ref = weakref.ref(stream) ++ + # Delete references to stream and connection + del stream + del connection +@@ -243,6 +246,14 @@ class TestAsyncClient(AsyncLocalServerTestBase): + status_code = await collect_task + self.assertEqual(200, status_code) + ++ # On free-threaded builds the stream's last reference is dropped on the ++ # native thread, and CPython defers freeing it to this (owning) thread. ++ # Run bytecode until that happens: the stream keeps the keep-alive ++ # connection open, and _stop_server() would otherwise block forever. ++ deadline = time.monotonic() + self.timeout ++ while stream_ref() is not None and time.monotonic() < deadline: ++ await asyncio.sleep(0.01) ++ + finally: + self._stop_server() + +diff --git a/test/test_http_client.py b/test/test_http_client.py +index 5364d7e..b2733d4 100644 +--- a/test/test_http_client.py ++++ b/test/test_http_client.py +@@ -3,6 +3,7 @@ + + import time + import socket ++import weakref + import sys + import subprocess + from urllib.parse import urlparse +@@ -267,6 +268,7 @@ class TestClient(LocalServerTestBase): + stream = connection.request(request) + stream.activate() + completion_future = stream.completion_future ++ stream_ref = weakref.ref(stream) + + # delete all local references + del stream +@@ -275,6 +277,14 @@ class TestClient(LocalServerTestBase): + # stream should still complete successfully + completion_future.result(self.timeout) + ++ # On free-threaded builds the stream's last reference is dropped on the ++ # native thread, and CPython defers freeing it to this (owning) thread. ++ # Run bytecode until that happens: the stream keeps the keep-alive ++ # connection open, and _stop_server() would otherwise block forever. ++ deadline = time.monotonic() + self.timeout ++ while stream_ref() is not None and time.monotonic() < deadline: ++ time.sleep(0.01) ++ + finally: + self._stop_server() + diff --git a/patches/awscrt/0.36.4/0001-test-let-the-stream-be-freed-before-stopping-the-local-server.patch b/patches/awscrt/0.36.4/0001-test-let-the-stream-be-freed-before-stopping-the-local-server.patch new file mode 100644 index 0000000000..647b8301fc --- /dev/null +++ b/patches/awscrt/0.36.4/0001-test-let-the-stream-be-freed-before-stopping-the-local-server.patch @@ -0,0 +1,118 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 25 Sep 2026 00:00:00 +0000 +Subject: [PATCH] test: let the stream be freed before stopping the local + server on free-threaded builds + +TestClient and TestAsyncClient's _test_stream_lives_until_complete drop +every local reference to the stream and the connection, wait for the +response, then call _stop_server(). That calls HTTPServer.shutdown(), and +the single-threaded HTTPServer is still inside handle() for the HTTP/1.1 +keep-alive connection. So shutdown() only returns once the client socket +closes, which happens only when the Python stream (which holds the +connection) is deallocated. + +With the GIL, the stream's last reference is dropped by the DECREFs in +s_on_stream_complete on the native event-loop thread, and the stream, +connection and socket are freed right there. On a free-threaded build, an +object's owner thread is the one that created it (here the test's main +thread). A DECREF from any other thread that takes the shared refcount +negative only queues the object for its owner to merge. The owner merges +the next time it runs bytecode, i.e. when the eval breaker fires. If the main +thread is already parked in shutdown()'s Event.wait() by then, nothing ever +merges the refcount. The stream never dies, the socket never closes and +the job hangs until the CI timeout. Whether it hangs depends on whether +the main thread gets from the completion wake-up into shutdown() before +the native thread's final DECREF. + +That is what happened on this repo's riscv64 runners. 0.36.4 cp314t hung in +test_http_client's test_stream_lives_until_complete_http, and on a +later run 0.36.3 cp314t hung in test_aiohttp_client's. Each version +passes on other runs, and test/ is byte-identical across 0.36.3, 0.36.4 +and 0.37.0. It is not specific to riscv64: with upstream's own +awscrt 0.36.3 cp314-cp314t x86_64 wheel from PyPI, a reproducer that +reaches shutdown() before the response arrives hangs 3/3 on 3.14.7t, +while cp314 returns in 0.5s 3/3. + +Keep a weakref to the stream and, after the completion check, run +bytecode until it is freed (bounded by the test's own timeout) before +stopping the server. Nothing else changes: the tests still check that the +stream completes after every local reference is gone, and now also +wait for it to be released instead of relying on the GIL's immediate +deallocation. + +Upstream-Status: To upstream [latent free-threading deadlock in upstream's own test harness, reproduced on x86_64 with upstream's PyPI wheel; not yet sent, since that needs a contributor-signed upstream PR, which this repo's automation does not open] + +Signed-off-by: Ludovic Henry +--- +diff --git a/test/test_aiohttp_client.py b/test/test_aiohttp_client.py +index 8d24c39..57facfb 100644 +--- a/test/test_aiohttp_client.py ++++ b/test/test_aiohttp_client.py +@@ -3,6 +3,7 @@ + + import time + import socket ++import weakref + import sys + import asyncio + import unittest +@@ -235,6 +236,8 @@ class TestAsyncClient(AsyncLocalServerTestBase): + # Schedule task to collect response but don't await it yet + collect_task = asyncio.create_task(response.collect_response(stream)) + ++ stream_ref = weakref.ref(stream) ++ + # Delete references to stream and connection + del stream + del connection +@@ -243,6 +246,14 @@ class TestAsyncClient(AsyncLocalServerTestBase): + status_code = await collect_task + self.assertEqual(200, status_code) + ++ # On free-threaded builds the stream's last reference is dropped on the ++ # native thread, and CPython defers freeing it to this (owning) thread. ++ # Run bytecode until that happens: the stream keeps the keep-alive ++ # connection open, and _stop_server() would otherwise block forever. ++ deadline = time.monotonic() + self.timeout ++ while stream_ref() is not None and time.monotonic() < deadline: ++ await asyncio.sleep(0.01) ++ + finally: + self._stop_server() + +diff --git a/test/test_http_client.py b/test/test_http_client.py +index 5364d7e..b2733d4 100644 +--- a/test/test_http_client.py ++++ b/test/test_http_client.py +@@ -3,6 +3,7 @@ + + import time + import socket ++import weakref + import sys + import subprocess + from urllib.parse import urlparse +@@ -267,6 +268,7 @@ class TestClient(LocalServerTestBase): + stream = connection.request(request) + stream.activate() + completion_future = stream.completion_future ++ stream_ref = weakref.ref(stream) + + # delete all local references + del stream +@@ -275,6 +277,14 @@ class TestClient(LocalServerTestBase): + # stream should still complete successfully + completion_future.result(self.timeout) + ++ # On free-threaded builds the stream's last reference is dropped on the ++ # native thread, and CPython defers freeing it to this (owning) thread. ++ # Run bytecode until that happens: the stream keeps the keep-alive ++ # connection open, and _stop_server() would otherwise block forever. ++ deadline = time.monotonic() + self.timeout ++ while stream_ref() is not None and time.monotonic() < deadline: ++ time.sleep(0.01) ++ + finally: + self._stop_server() + diff --git a/patches/awscrt/0.37.0/0001-test-raise-appexit_http-timeout-for-slow-shared-runners.patch b/patches/awscrt/0.37.0/0001-test-raise-appexit_http-timeout-for-slow-shared-runners.patch new file mode 100644 index 0000000000..c1baa70f2a --- /dev/null +++ b/patches/awscrt/0.37.0/0001-test-raise-appexit_http-timeout-for-slow-shared-runners.patch @@ -0,0 +1,63 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Thu, 24 Sep 2026 00:00:00 +0000 +Subject: [PATCH] test: raise appexit_http's TIMEOUT for slow/shared runners + +test_appexit.test_http runs test/appexit_http.py as a subprocess once per +Stage, asserting each one exits 0. Every wait in that script - the HTTP +connect, the body callback, the connection shutdown future, and both +ClientBootstrap/EventLoopGroup shutdown_event.wait() calls - shares one +hardcoded TIMEOUT = 30.0. + +On this repo's riscv64 runners, the cp314-cp314t leg of awscrt 0.37.0 failed +at exactly the ClientBootstrapDone stage: + + File "test/appexit_http.py", line 139, in + assert bootstrap_shutdown_event.wait(TIMEOUT) + AssertionError + +The interleaved trace-level log (init_logging(Trace) writes to the same +stdout the subprocess inherits) shows why this is a margin problem, not a +hang: the host resolver's own maintenance loop re-resolves s3.amazonaws.com +roughly once a second and only decides to kill its background thread ("no +requests have been made for an address ... for the duration of the ttl, or +this thread is being forcibly shutdown") about 24 seconds after the +connection is torn down, before the channel-bootstrap release / event-loop +destroy sequence that fires bootstrap_shutdown_event can even begin. That +leaves under 6 seconds of the 30-second budget for the rest of native +teardown to reach Python and set the event - on a shared riscv64 runner +(other build jobs routinely run concurrently on the same host, see gotcha +316 in the RISE python-wheels skill) driving a free-threaded (no-GIL) +interpreter, whose per-object atomic refcounting adds measurable overhead to +exactly this kind of native-callback-into-Python teardown chain. + +Neither of the two source changes between 0.36.4 (whose cp314-cp314t leg +passed on the same runners in the same run) and 0.37.0 - a handful of small +aws-c-common commits and a version bump each of aws-c-s3 and aws-lc - touch +the ClientBootstrap/event-loop/host-resolver shutdown path (aws-c-io itself +is unchanged), and test/appexit_http.py and test/test_appexit.py are +byte-identical across 0.36.3, 0.36.4 and 0.37.0. So this a pre-existing +margin that 0.37.0 happened to tip over on this run, not a version-specific +regression, and not specific to riscv64 either - it is a statement about how +long this host takes to tear the stack down under load. Tripling TIMEOUT +to 90 seconds gives real headroom for a busy shared runner and free-threaded +overhead while still failing fast (well inside any CI job timeout) if the +shutdown genuinely never completes. + +Upstream-Status: Inappropriate [CI-timing headroom for this repo's shared riscv64 runners; not a functional defect and not worth carrying upstream] + +Signed-off-by: Ludovic Henry +--- +diff --git a/test/appexit_http.py b/test/appexit_http.py +index d0ee65d..3b31503 100644 +--- a/test/appexit_http.py ++++ b/test/appexit_http.py +@@ -33,7 +33,7 @@ Stage = enum.Enum('Stage', [ + + + EXIT_STAGE = Stage.Done +-TIMEOUT = 30.0 ++TIMEOUT = 90.0 + REQUEST_HOST = 's3.amazonaws.com' + REQUEST_PATH = '/code-sharing-aws-crt/elastigirl.png' + REQUEST_PORT = 443 diff --git a/patches/awscrt/0.37.0/0002-test-let-the-stream-be-freed-before-stopping-the-local-server.patch b/patches/awscrt/0.37.0/0002-test-let-the-stream-be-freed-before-stopping-the-local-server.patch new file mode 100644 index 0000000000..647b8301fc --- /dev/null +++ b/patches/awscrt/0.37.0/0002-test-let-the-stream-be-freed-before-stopping-the-local-server.patch @@ -0,0 +1,118 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 25 Sep 2026 00:00:00 +0000 +Subject: [PATCH] test: let the stream be freed before stopping the local + server on free-threaded builds + +TestClient and TestAsyncClient's _test_stream_lives_until_complete drop +every local reference to the stream and the connection, wait for the +response, then call _stop_server(). That calls HTTPServer.shutdown(), and +the single-threaded HTTPServer is still inside handle() for the HTTP/1.1 +keep-alive connection. So shutdown() only returns once the client socket +closes, which happens only when the Python stream (which holds the +connection) is deallocated. + +With the GIL, the stream's last reference is dropped by the DECREFs in +s_on_stream_complete on the native event-loop thread, and the stream, +connection and socket are freed right there. On a free-threaded build, an +object's owner thread is the one that created it (here the test's main +thread). A DECREF from any other thread that takes the shared refcount +negative only queues the object for its owner to merge. The owner merges +the next time it runs bytecode, i.e. when the eval breaker fires. If the main +thread is already parked in shutdown()'s Event.wait() by then, nothing ever +merges the refcount. The stream never dies, the socket never closes and +the job hangs until the CI timeout. Whether it hangs depends on whether +the main thread gets from the completion wake-up into shutdown() before +the native thread's final DECREF. + +That is what happened on this repo's riscv64 runners. 0.36.4 cp314t hung in +test_http_client's test_stream_lives_until_complete_http, and on a +later run 0.36.3 cp314t hung in test_aiohttp_client's. Each version +passes on other runs, and test/ is byte-identical across 0.36.3, 0.36.4 +and 0.37.0. It is not specific to riscv64: with upstream's own +awscrt 0.36.3 cp314-cp314t x86_64 wheel from PyPI, a reproducer that +reaches shutdown() before the response arrives hangs 3/3 on 3.14.7t, +while cp314 returns in 0.5s 3/3. + +Keep a weakref to the stream and, after the completion check, run +bytecode until it is freed (bounded by the test's own timeout) before +stopping the server. Nothing else changes: the tests still check that the +stream completes after every local reference is gone, and now also +wait for it to be released instead of relying on the GIL's immediate +deallocation. + +Upstream-Status: To upstream [latent free-threading deadlock in upstream's own test harness, reproduced on x86_64 with upstream's PyPI wheel; not yet sent, since that needs a contributor-signed upstream PR, which this repo's automation does not open] + +Signed-off-by: Ludovic Henry +--- +diff --git a/test/test_aiohttp_client.py b/test/test_aiohttp_client.py +index 8d24c39..57facfb 100644 +--- a/test/test_aiohttp_client.py ++++ b/test/test_aiohttp_client.py +@@ -3,6 +3,7 @@ + + import time + import socket ++import weakref + import sys + import asyncio + import unittest +@@ -235,6 +236,8 @@ class TestAsyncClient(AsyncLocalServerTestBase): + # Schedule task to collect response but don't await it yet + collect_task = asyncio.create_task(response.collect_response(stream)) + ++ stream_ref = weakref.ref(stream) ++ + # Delete references to stream and connection + del stream + del connection +@@ -243,6 +246,14 @@ class TestAsyncClient(AsyncLocalServerTestBase): + status_code = await collect_task + self.assertEqual(200, status_code) + ++ # On free-threaded builds the stream's last reference is dropped on the ++ # native thread, and CPython defers freeing it to this (owning) thread. ++ # Run bytecode until that happens: the stream keeps the keep-alive ++ # connection open, and _stop_server() would otherwise block forever. ++ deadline = time.monotonic() + self.timeout ++ while stream_ref() is not None and time.monotonic() < deadline: ++ await asyncio.sleep(0.01) ++ + finally: + self._stop_server() + +diff --git a/test/test_http_client.py b/test/test_http_client.py +index 5364d7e..b2733d4 100644 +--- a/test/test_http_client.py ++++ b/test/test_http_client.py +@@ -3,6 +3,7 @@ + + import time + import socket ++import weakref + import sys + import subprocess + from urllib.parse import urlparse +@@ -267,6 +268,7 @@ class TestClient(LocalServerTestBase): + stream = connection.request(request) + stream.activate() + completion_future = stream.completion_future ++ stream_ref = weakref.ref(stream) + + # delete all local references + del stream +@@ -275,6 +277,14 @@ class TestClient(LocalServerTestBase): + # stream should still complete successfully + completion_future.result(self.timeout) + ++ # On free-threaded builds the stream's last reference is dropped on the ++ # native thread, and CPython defers freeing it to this (owning) thread. ++ # Run bytecode until that happens: the stream keeps the keep-alive ++ # connection open, and _stop_server() would otherwise block forever. ++ deadline = time.monotonic() + self.timeout ++ while stream_ref() is not None and time.monotonic() < deadline: ++ time.sleep(0.01) ++ + finally: + self._stop_server() +