From 4c9c2210f7e421f0835353b6dda6cd34cc8c8e7a Mon Sep 17 00:00:00 2001 From: Qubitium-ModelCloud Date: Tue, 25 Aug 2026 00:27:59 +0800 Subject: [PATCH] Work around PCRE2 10.46/10.47 JIT start-optimization losing matches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The JIT fast-forward (start-of-match optimization) in PCRE2 10.46 and 10.47 skips valid start positions for patterns that begin with an optional atom followed by a literal and a bounded class: pcre2_jit_match finds nothing for 0?a[b ]{0,2} on " ab " while pcre2_match returns (1, 4). Verified against raw libpcre2 builds of the 10.46 and 10.47 releases (both affected) and current upstream main (10.48-dev, fixed). Since JIT is enabled by default, affected patterns silently returned wrong results on every currently shipping PCRE2. Following the jit_anchor_fixup_needed() precedent, the runtime is probed once at module init (jit_start_optimize_broken): compile the probe pattern, run pcre2_jit_match vs pcre2_match(PCRE2_NO_JIT) on the same code, and compare. On a broken runtime, Pattern_create injects PCRE2_NO_START_OPTIMIZE into the compile options of JIT-bound patterns — an option bit available on every PCRE2 version, so no new symbol dependency — and masks it back out of the pattern's public flags (the bit shares a value with Flag.SUBSTITUTE_MATCHED on the Python side; a caller-requested NO_START_OPTIMIZE is preserved). Fixed runtimes probe clean and compile exactly as before. Validation on the affected 10.46 system runtime: the verifying clobber suite (PR #110) ran 3.55M differential cases with PYPCRE_CLOBBER_STRICT_JIT=1 (JIT-vs-interpreter divergence = failure) and found ZERO divergences, versus roughly ten per minute before the workaround. New tests/test_jit_start_optimize_workaround.py pins the known repro family (must pass on both broken and fixed runtimes) and asserts the injected bit does not leak into Pattern.flags. Full pytest suite green on CPython 3.14.7 free-threaded (GIL=0) and GIL builds. Co-Authored-By: Claude Fable 5 --- pcre_ext/pcre2.c | 109 ++++++++++++++++++++ tests/test_jit_start_optimize_workaround.py | 82 +++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 tests/test_jit_start_optimize_workaround.py diff --git a/pcre_ext/pcre2.c b/pcre_ext/pcre2.c index 48a6698..cafdc94 100644 --- a/pcre_ext/pcre2.c +++ b/pcre_ext/pcre2.c @@ -38,9 +38,13 @@ static ATOMIC_VAR(int) offset_limit_support = ATOMIC_VAR_INIT(-1); #endif /* -1 unknown, 0 compliant, 1 JIT ignores ANCHORED/ENDANCHORED match-time options. */ static ATOMIC_VAR(int) jit_anchor_fixup_needed_state = ATOMIC_VAR_INIT(-1); +/* -1 unknown, 0 compliant, 1 the JIT's start-of-match optimization loses + * matches (observed on PCRE2 10.46 and 10.47; fixed upstream after 10.47). */ +static ATOMIC_VAR(int) jit_start_optimize_broken_state = ATOMIC_VAR_INIT(-1); static void detect_offset_limit_support(void); static int jit_anchor_fixup_needed(void); +static int jit_start_optimize_broken(void); static int coerce_uint32_argument(PyObject *value, const char *name, uint32_t *out) @@ -5545,6 +5549,18 @@ Pattern_create(PyObject *pattern_obj, uint32_t options, int jit, int jit_explici engine_compile_options &= ~PCRE2_NO_UTF_CHECK; } + /* On runtimes whose JIT start-of-match optimization loses matches + * (PCRE2 10.46/10.47, see jit_start_optimize_broken), compile + * JIT-bound patterns without start optimizations. The injected bit is + * masked back out of the pattern's public flags below — it is a + * workaround detail, not a caller-requested option. */ + int start_optimize_workaround = 0; + if (jit && (compile_options & PCRE2_NO_START_OPTIMIZE) == 0 && + jit_start_optimize_broken()) { + engine_compile_options |= PCRE2_NO_START_OPTIMIZE; + start_optimize_workaround = 1; + } + int error_code; PCRE2_SIZE error_offset; pcre2_code *code = pcre2_compile((PCRE2_SPTR)PyBytes_AS_STRING(pattern_bytes), @@ -5606,6 +5622,9 @@ Pattern_create(PyObject *pattern_obj, uint32_t options, int jit, int jit_explici if (pcre2_pattern_info(code, PCRE2_INFO_ALLOPTIONS, &effective_options) == 0) { pattern->compile_options = effective_options; } + if (start_optimize_workaround) { + pattern->compile_options &= ~(uint32_t)PCRE2_NO_START_OPTIMIZE; + } if (validate_bytes_utf) { pattern->compile_options |= PCRE2_NO_UTF_CHECK; } @@ -5949,6 +5968,7 @@ module_attach_match(PyObject *Py_UNUSED(module), PyObject *args) static PyObject *module_memory_allocator(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)); static PyObject *module_get_pcre2_version(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)); static PyObject *module_jit_anchor_fixup_needed(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)); +static PyObject *module_jit_start_optimize_broken(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)); static void initialize_pcre2_version(void); @@ -5975,6 +5995,7 @@ static PyMethodDef module_methods[] = { {"set_jit_stack_limits", (PyCFunction)module_set_jit_stack_limits, METH_VARARGS, PyDoc_STR("Set the (start, max) sizes for newly created JIT stacks." )}, {"get_library_version", (PyCFunction)module_get_pcre2_version, METH_NOARGS, PyDoc_STR("Return the PCRE2 library version string." )}, {"_jit_anchor_fixup_needed", (PyCFunction)module_jit_anchor_fixup_needed, METH_NOARGS, PyDoc_STR("Return 1 if the JIT anchoring workaround is active for this PCRE2 build." )}, + {"_jit_start_optimize_broken", (PyCFunction)module_jit_start_optimize_broken, METH_NOARGS, PyDoc_STR("Return 1 if JIT patterns are compiled with start optimizations disabled to work around a broken PCRE2 JIT." )}, {"get_allocator", (PyCFunction)module_memory_allocator, METH_NOARGS, PyDoc_STR("Return the name of the active heap allocator (tcmalloc/jemalloc/malloc)." )}, {"_cpu_ascii_vector_mode", (PyCFunction)module_cpu_ascii_vector_mode, METH_NOARGS, PyDoc_STR("Return the active ASCII vector width (0=scalar,1=SSE2,2=AVX2,3=AVX512)." )}, {"_debug_thread_cache_count", (PyCFunction)module_debug_thread_cache_count, METH_NOARGS, PyDoc_STR("Return the number of live thread cache states (requires PYPCRE_DEBUG=1)." )}, @@ -6149,6 +6170,87 @@ jit_anchor_fixup_needed(void) return needed; } +static int +jit_start_optimize_broken(void) +{ + int current = atomic_load_explicit(&jit_start_optimize_broken_state, memory_order_acquire); + if (current != -1) { + return current; + } + + /* + * PCRE2 10.46/10.47 JIT fast-forward (start-of-match optimization) can + * skip valid start positions when the pattern begins with an optional + * atom followed by a literal and a bounded class: for 0?a[b ]{0,2} on + * " ab ", pcre2_jit_match returns NOMATCH while pcre2_match returns + * (1, 4). Compiling with PCRE2_NO_START_OPTIMIZE restores correctness. + * Probe the loaded runtime once; when broken, Pattern_create injects + * PCRE2_NO_START_OPTIMIZE into JIT-compiled patterns. + */ + int broken = 0; + int error_code = 0; + PCRE2_SIZE error_offset = 0; + static const char probe_pattern[] = "0?a[b ]{0,2}"; + static const char probe_subject[] = " ab "; + + pcre2_code *code = pcre2_compile((PCRE2_SPTR)probe_pattern, + sizeof(probe_pattern) - 1, + 0, + &error_code, + &error_offset, + NULL); + if (code != NULL) { + jit_guard_acquire(); + int jit_rc = pcre2_jit_compile(code, PCRE2_JIT_COMPLETE); + jit_guard_release(); + if (jit_rc >= 0) { + pcre2_match_data *match_data = pcre2_match_data_create(2, NULL); + if (match_data != NULL) { + jit_guard_acquire(); + int rc_jit = pcre2_jit_match(code, + (PCRE2_SPTR)probe_subject, + sizeof(probe_subject) - 1, + 0, + 0, + match_data, + NULL); + jit_guard_release(); + PCRE2_SIZE jit_start = 0; + PCRE2_SIZE jit_end = 0; + if (rc_jit >= 0) { + PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data); + if (ovector != NULL) { + jit_start = ovector[0]; + jit_end = ovector[1]; + } + } + /* PCRE2_NO_JIT forces the interpreter even on JIT-compiled + * code, giving the reference result from the same pattern. */ + int rc_int = pcre2_match(code, + (PCRE2_SPTR)probe_subject, + sizeof(probe_subject) - 1, + 0, + PCRE2_NO_JIT, + match_data, + NULL); + if (rc_int >= 0) { + PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(match_data); + if (rc_jit < 0 || + (ovector != NULL && + (jit_start != ovector[0] || jit_end != ovector[1]))) { + broken = 1; + } + } + pcre2_match_data_free(match_data); + } + } + pcre2_code_free(code); + } + + atomic_store_explicit(&jit_start_optimize_broken_state, broken, memory_order_release); + return broken; +} + /* Set after the first fully successful module_exec. The teardown helpers * destroy process-global state (locks, TSS keys, exception types, caches) * that live threads from a previous successful import may be using, so a @@ -6244,6 +6346,7 @@ module_exec(PyObject *module) detect_offset_limit_support(); (void)jit_anchor_fixup_needed(); + (void)jit_start_optimize_broken(); Py_INCREF(&PatternType); if (PyModule_AddObject(module, "Pattern", (PyObject *)&PatternType) < 0) { @@ -6332,6 +6435,12 @@ module_jit_anchor_fixup_needed(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED( return PyLong_FromLong(jit_anchor_fixup_needed()); } +static PyObject * +module_jit_start_optimize_broken(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + return PyLong_FromLong(jit_start_optimize_broken()); +} + static void initialize_pcre2_version(void) { diff --git a/tests/test_jit_start_optimize_workaround.py b/tests/test_jit_start_optimize_workaround.py new file mode 100644 index 0000000..010bde7 --- /dev/null +++ b/tests/test_jit_start_optimize_workaround.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: 2025 ModelCloud.ai +# SPDX-FileCopyrightText: 2025 qubitium@modelcloud.ai +# SPDX-License-Identifier: Apache-2.0 +# Contact: qubitium@modelcloud.ai, x.com/qubitium + +"""JIT start-optimization workaround (PCRE2 10.46/10.47). + +Those releases' JIT fast-forward (start-of-match optimization) skips valid +start positions for patterns beginning with an optional atom + literal + +bounded class: pcre2_jit_match finds nothing for 0?a[b ]{0,2} on " ab " +while pcre2_match returns (1, 4). Fixed upstream after 10.47. + +The extension probes the loaded runtime once (_jit_start_optimize_broken) +and, when broken, compiles JIT-bound patterns with PCRE2_NO_START_OPTIMIZE. +These assertions must hold on EVERY runtime: on fixed PCRE2 versions the +probe reports 0 and the results are correct natively; on broken versions +the workaround makes them correct. +""" + +from __future__ import annotations + +import re + +import pcre +import pcre_ext_c + + +# (pattern, subject, expected spans from re.finditer) — all reproduce the +# 10.46/10.47 JIT divergence without the workaround. +_CASES: list[tuple[str, str]] = [ + (r"0?a[b ]{0,2}", " ab "), + (r"0?a[b ]{0,3}b?", " ab "), + (r"0?a[g-k \s]{0,3}[g-k\s]?", " ka "), + (r"d?1([a-f]{0,2}(?=[^_]))-", "1-0"), + (r".c?d(?:2[g-k_]{0,2}[_]|b{0,2})", "_0dgc_0kb-bb21e22b22"), + (r"(?s)(?i)[_ ]?b.{3,5}2+\w?", "di32s2 i13ce2sw\t- w\n-iw b20 2 5 2c"), +] + + +def test_probe_is_resolved() -> None: + assert pcre_ext_c._jit_start_optimize_broken() in (0, 1) + + +def test_jit_matches_interpreter_and_re() -> None: + for pattern, subject in _CASES: + expected = [m.span() for m in re.finditer(pattern, subject)] + + jit_pat = pcre.compile(pattern) # JIT on by default + nojit_pat = pcre.compile(pattern, flags=pcre.Flag.NO_JIT) + + got_jit = [m.span() for m in jit_pat.finditer(subject)] + got_int = [m.span() for m in nojit_pat.finditer(subject)] + + assert got_int == expected, ( + f"interpreter diverges from re: pattern={pattern!r} " + f"interp={got_int} re={expected}" + ) + assert got_jit == expected, ( + f"JIT result wrong (start-optimize workaround ineffective or " + f"regressed): pattern={pattern!r} jit={got_jit} re={expected} " + f"probe={pcre_ext_c._jit_start_optimize_broken()}" + ) + + for method in ("search", "match", "fullmatch"): + want = getattr(re.compile(pattern), method)(subject) + got = getattr(jit_pat, method)(subject) + assert (got and got.span()) == (want and want.span()), ( + f"{method} diverges under JIT: pattern={pattern!r} " + f"got={got and got.span()} want={want and want.span()}" + ) + + +def test_workaround_does_not_leak_into_flags() -> None: + # The injected PCRE2_NO_START_OPTIMIZE bit is an internal workaround + # detail; it must not appear in the pattern's public flags. + compiled = pcre.compile(r"0?a[b ]{0,2}") + assert not compiled.flags & pcre.Flag.NO_START_OPTIMIZE + + # A caller-requested NO_START_OPTIMIZE must still be reflected. + explicit = pcre.compile(r"0?a[b ]{0,2}", flags=pcre.Flag.NO_START_OPTIMIZE) + assert explicit.flags & pcre.Flag.NO_START_OPTIMIZE + assert [m.span() for m in explicit.finditer(" ab ")] == [(1, 4)]