diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c779832..e494646 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,37 @@ jobs: flags: ${{ matrix.os }}-py${{ matrix.python-version }} fail_ci_if_error: false + # 依赖时序的断言只会**随机**报红:main 上 Python 3.11 (macos-latest) 就因为 + # "睡 0.3 秒后至少 3 次检查"拿到 2 而红过一次——不是产品代码错,而是断言考的是 + # "这台机器够快"。这条腿把那种机器搬进 CI:往工作线程注入延迟(见 + # tests/conftest.py),于是这类测试**当场**失败,而不是某天随机变红。 + # + # 只跑一个 OS/版本:它的价值在确定性,不在覆盖率——版本/平台差异由上面的矩阵负责。 + timing: + name: Timing guard (injected thread latency) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install project with dev deps + run: python -m pip install --upgrade pip && pip install -e ".[dev]" + + # 故意不加 `-q`:报告头里的那行横幅就是"守卫真的开着"的证据,下一行 grep 它。 + # 一条静默失效的注入会把这条腿变成空跑,而输出仍然是绿的。 + - name: Run tests with injected latency + env: + PONTE_TEST_THREAD_DELAY: "0.15" + run: | + set -o pipefail + pytest 2>&1 | tee slow-runner.log + grep -q "thread latency: +0.15s" slow-runner.log + build: name: Build & verify wheel runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8f21e7d..ef94869 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,6 +36,15 @@ CI runs lint + types on Linux, the same test suite on Windows / Linux / macOS × Python 3.11 / 3.12, and a `build` job that installs the wheel and runs `ponte init`. Coverage is uploaded to Codecov. +One more job re-runs the suite with `PONTE_TEST_THREAD_DELAY=0.15`. That variable +makes `tests/conftest.py` inject latency into worker-thread sleeps and waits, so a +test that only passes because the machine is fast fails there **every time** +instead of flaking once in a while. Run it the same way before blaming a runner: + +```bash +PONTE_TEST_THREAD_DELAY=0.15 pytest # 慢机器模拟(Git Bash / POSIX 语法) +``` + ## Project layout ``` @@ -152,6 +161,15 @@ Linux 上跑 lint + 类型检查,在 Windows / Linux / macOS × Python 3.11 / 上跑同一套测试,另有 `build` 任务会安装 wheel 并执行 `ponte init`。覆盖率 上报到 Codecov。 +还有一个任务会用 `PONTE_TEST_THREAD_DELAY=0.15` 再跑一遍:这个变量让 +`tests/conftest.py` 往工作线程的 sleep/wait 里注入延迟,于是“只有机器够快才 +通过”的测试会**每次都**在那里失败,而不是偶发地红一次。怀疑是 runner 抽风之前, +先这样在本地跑一遍: + +```bash +PONTE_TEST_THREAD_DELAY=0.15 pytest # 模拟慢机器(Git Bash / POSIX 语法) +``` + ## 项目结构 ``` diff --git a/README.md b/README.md index 3f80e81..2806cdb 100644 --- a/README.md +++ b/README.md @@ -325,7 +325,11 @@ python _smoke_test.py # zero-dependency quick check CI runs lint + types on Linux, and the test suite across Windows/Linux/macOS × Python 3.11/3.12, reporting coverage to -[Codecov](https://codecov.io/gh/modusensus/ponte). A `build` job also installs +[Codecov](https://codecov.io/gh/modusensus/ponte). One extra job re-runs the +suite with `PONTE_TEST_THREAD_DELAY=0.15`: that injects latency into +worker-thread sleeps and waits, so a test that only passes on a fast machine +fails there *every* time instead of flaking once in a while. Set the same +variable locally to reproduce such a machine. A `build` job also installs the built wheel and runs `ponte init`, so a packaging regression cannot ship again. See [CONTRIBUTING.md](CONTRIBUTING.md). @@ -626,8 +630,11 @@ python _smoke_test.py # 零依赖快速自检 CI 在 Linux 上跑 lint + 类型检查,在 Windows/Linux/macOS × Python 3.11/3.12 上跑测试,覆盖率上报到 -[Codecov](https://codecov.io/gh/modusensus/ponte)。另有一个 `build` 任务会 -安装打好的 wheel 并执行 `ponte init`,避免打包问题再次溜进发布。详见 +[Codecov](https://codecov.io/gh/modusensus/ponte)。另有一个任务会用 +`PONTE_TEST_THREAD_DELAY=0.15` 再跑一遍:它往工作线程的 sleep/wait 里注入 +延迟,于是"只有机器够快才通过"的测试会**每次都**在那里失败,而不是偶发地 +红一次。本地设同一个变量即可复现这种机器。`build` 任务会安装打好的 wheel 并 +执行 `ponte init`,避免打包问题再次溜进发布。详见 [CONTRIBUTING.md](CONTRIBUTING.md)。 ## 📝 注意事项 diff --git a/tests/conftest.py b/tests/conftest.py index 1c27df7..4ad4356 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,18 +4,97 @@ test that sets ``--config``/``PONTE_CONFIG`` would leak that state into the next test, and ``get_config()`` could pick up the developer's real ``~/.config/ponte/config.toml``. + +It also owns the optional *latency injection* that turns "this test only passes +on a fast machine" from a random CI failure into a deterministic one — see +:func:`_install_thread_delay` and ``PONTE_TEST_THREAD_DELAY``. """ from __future__ import annotations +import os +import threading +import time + import pytest from ponte import config as config_module +#: Seconds added to every *worker-thread* sleep/wait. Unset or ``0`` disables the +#: injection entirely, so the normal suite is untouched. +_THREAD_DELAY_ENV = "PONTE_TEST_THREAD_DELAY" + + +def _thread_delay() -> float: + """Seconds to inject into worker-thread sleeps and waits (``0`` = off).""" + raw = os.environ.get(_THREAD_DELAY_ENV, "").strip() + if not raw: + return 0.0 + try: + delay = float(raw) + except ValueError: + raise SystemExit( + f"{_THREAD_DELAY_ENV}={raw!r} is not a number. Refusing to continue: " + "this switch exists to make timing assumptions fail, so silently " + "disabling it would hide exactly what it guards." + ) from None + if delay < 0: + raise SystemExit(f"{_THREAD_DELAY_ENV} must not be negative: {raw!r}") + return delay + + +def _install_thread_delay(delay: float) -> None: + """Make the code under test slower than the test that watches it. + + "Slow machine" is never uniform. Every wait and sleep a *worker* performs + gets queued behind whatever else the runner is doing, while the test's own + ``time.sleep(0.3)`` stays exactly 0.3s — that asymmetry is what makes + "slept 0.3s, expected 3 checks" fail on a loaded macOS runner and pass here. + So the injection only slows threads that are not the main thread; stretching + both sides equally would preserve every ratio and prove nothing (the test's + nap would grow to 0.9s and the loop's ``wait(0.05)`` to 0.2s, still fitting + six checks). + + Two primitives cover how a background thread paces itself: ``Event.wait`` + and ``time.sleep``. Clocks (``time.monotonic``) are deliberately left alone — + moving those would corrupt deadlines instead of emulating load. A ``sleep(0)`` + is a yield rather than a wait, and load does not stretch it, so it stays. + + Scope, honestly: this emulates *a worker that waits or is scheduled late*, + which is the failure mode behind the CI flake this exists for. It does not + emulate *a worker starved of CPU* — a test that naps on the main thread while + a worker does real work is untouched by it and still relies on the runner + being fast. Don't read a green run here as "no timing assumptions left". + """ + main_thread = threading.main_thread() + real_sleep = time.sleep + real_wait = threading.Event.wait + + def slow_sleep(seconds: float) -> None: + if seconds > 0 and threading.current_thread() is not main_thread: + seconds += delay + real_sleep(seconds) + + def slow_wait(event: threading.Event, timeout: float | None = None) -> bool: + if timeout is not None and threading.current_thread() is not main_thread: + timeout += delay + return real_wait(event, timeout) + + time.sleep = slow_sleep + threading.Event.wait = slow_wait + + +_DELAY = _thread_delay() +if _DELAY: + _install_thread_delay(_DELAY) + def pytest_report_header() -> str: """Show where the config layer resolves to, which explains most failures.""" - return "ponte config search path: " + " | ".join(config_module.config_search_paths()) + header = "ponte config search path: " + " | ".join(config_module.config_search_paths()) + if _DELAY: + header += f"\nthread latency: +{_DELAY:g}s injected into every worker sleep/wait" + return header @pytest.fixture(autouse=True) diff --git a/tests/test_timing_guard.py b/tests/test_timing_guard.py new file mode 100644 index 0000000..23be2c8 --- /dev/null +++ b/tests/test_timing_guard.py @@ -0,0 +1,81 @@ +"""Guards for the latency injection itself (``PONTE_TEST_THREAD_DELAY``). + +一个静默失效的守卫比没有守卫更糟:它会把"慢 runner"那条 CI 腿变成一场空跑,而 +输出仍是绿的。所以这里直接测量机制,而不是相信开关的名字。 +""" + +from __future__ import annotations + +import os +import sys +import threading +import time + +import pytest + +#: Mirrors ``tests/conftest.py``; read from the environment so this file does not +#: depend on how pytest happens to name the conftest module. +_DELAY = float(os.environ.get("PONTE_TEST_THREAD_DELAY", "0") or 0) + +_PROBE_WAIT = 0.01 + + +def _wait_in_worker() -> float: + """Seconds a *worker* thread actually spends in ``Event.wait(0.01)``.""" + elapsed: list[float] = [] + + def worker() -> None: + started = time.monotonic() + threading.Event().wait(_PROBE_WAIT) + elapsed.append(time.monotonic() - started) + + thread = threading.Thread(target=worker, daemon=True) + thread.start() + thread.join(timeout=5) + assert elapsed, "worker thread never finished its wait" + return elapsed[0] + + +def test_injection_slows_workers_only_when_the_switch_is_on() -> None: + """开着时工作线程变慢、主线程不变;关着时两者都不受影响。 + + 主线程这一半是重点:把测试自己的 ``time.sleep`` 也拉长会保持所有比例, + 于是它测不出任何东西(见 conftest 里的说明)。 + """ + started = time.monotonic() + threading.Event().wait(_PROBE_WAIT) + main_elapsed = time.monotonic() - started + worker_elapsed = _wait_in_worker() + + # `is` 而不是 `==`:开关是"开/关"两种情形,不是一条连续刻度。 + if _DELAY: + assert worker_elapsed >= _PROBE_WAIT + _DELAY * 0.8, worker_elapsed + assert main_elapsed < _PROBE_WAIT + _DELAY * 0.5, main_elapsed + else: + assert worker_elapsed < _PROBE_WAIT + 0.25, worker_elapsed + assert main_elapsed < _PROBE_WAIT + 0.25, main_elapsed + + +def _conftest_header() -> str: + """The header our own conftest contributes, via the module pytest loaded.""" + # 不调 pytestconfig.hook:那个钩子是 firstresult,返回的是**别的插件**的结果; + # 也不 import:模块名取决于 pytest 的 import 模式,问 sys.modules 最稳。 + for name in ("conftest", "tests.conftest"): + module = sys.modules.get(name) + header = getattr(module, "pytest_report_header", None) + if header is not None: + return header() + pytest.skip("conftest module is not loaded under this import mode") + raise AssertionError("unreachable") # pragma: no cover - pytest.skip never returns + + +def test_injection_is_announced_in_the_report_header() -> None: + """横幅要出现在头部:静默生效的注入无法从日志里证明自己跑过。 + + 这条横幅也是 CI 那一步能自检的依据(job 会 grep 它,见 ci.yml)。 + """ + header = _conftest_header() + if _DELAY: + assert f"thread latency: +{_DELAY:g}s" in header, header + else: + assert "thread latency" not in header, header