SPOC-494: Introduce random delays injection for testing bgworker races

.github/workflows/random-delays-regress.yml

@@ -0,0 +1,72 @@
+name: Regression Tests with Random Delays
+run-name: Regression tests compiled with SPOCK_RANDOM_DELAYS
+
+# Runs the standard Spock regression suite against a build where every
+# BEGIN, worker start, and worker finish introduces a random 1–100 ms delay.
+# This exercises timing-sensitive paths that are invisible under normal load.
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  contents: read
+
+jobs:
+  regress-random-delays:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        pgver: [15, 16, 17, 18]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout spock
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Add permissions
+        run: |
+          sudo chmod -R a+w ${GITHUB_WORKSPACE}
+
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3
+
+      - name: Build docker image with SPOCK_RANDOM_DELAYS
+        run: |
+          docker build \
+            --build-arg PGVER=${{ matrix.pgver }} \
+            --build-arg SPOCK_RANDOM_DELAYS=1 \
+            -t spock-random-delays \
+            -f tests/docker/Dockerfile-step-1.el9 .
+        timeout-minutes: 30
+
+      - name: Run regression tests
+        run: |
+          REG_CT_NAME="spock-random-delays-${{ matrix.pgver }}-${{ github.run_id }}-${{ github.run_attempt }}"
+          echo "REG_CT_NAME=$REG_CT_NAME" >> "$GITHUB_ENV"
+          docker run --name "$REG_CT_NAME" \
+            -e PGVER=${{ matrix.pgver }} \
+            spock-random-delays \
+            /home/pgedge/run-spock-regress.sh
+        timeout-minutes: 60
+
+      - name: Collect regression artifacts
+        if: ${{ always() }}
+        run: |
+          docker cp "$REG_CT_NAME":/home/pgedge/spock/tests/regress/regression_output \
+            "${GITHUB_WORKSPACE}/tests/regress/" || true
+          docker rm -f "$REG_CT_NAME" || true
+
+      - name: Upload regression artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: random-delays-regress-pg${{ matrix.pgver }}
+          path: tests/regress/regression_output/**
+          if-no-files-found: ignore
+          retention-days: 7

Makefile

@@ -22,6 +22,13 @@ PG_CPPFLAGS += -I$(libpq_srcdir) \
 			   -I"$(realpath include)" \
 			   -I"$(realpath src/compat/$(PGVER))" \
 			   -Werror=implicit-function-declaration
+
+# When SPOCK_RANDOM_DELAYS is set in the environment, inject unconditional
+# random delays at worker start/finish without requiring injection point
+# infrastructure or a runtime spock.inject_attach() call.
+ifdef SPOCK_RANDOM_DELAYS
+PG_CPPFLAGS += -DSPOCK_RANDOM_DELAYS
+endif
 SHLIB_LINK += $(libpq) $(filter -lintl, $(LIBS))
 
 REGRESS := __placeholder__

include/spock_injection.h

@@ -0,0 +1,38 @@
+/*-------------------------------------------------------------------------
+ *
+ * spock_injection.h
+ *		Injection point support for the Spock extension.
+ *
+ * Defines SPOCK_WORKER_DELAY(), placed at worker start/finish sites:
+ *
+ *   SPOCK_RANDOM_DELAYS defined  – calls spock_random_delay() directly;
+ *                                   fires unconditionally, no runtime setup.
+ *   PG17+ without the flag       – expands to INJECTION_POINT(); the core
+ *                                   injection_points module can attach to
+ *                                   'spock-worker-delay' when needed.
+ *   pre-PG17 without the flag    – compiles to nothing.
+ *
+ * Copyright (c) 2022-2026, pgEdge, Inc.
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef SPOCK_INJECTION_H
+#define SPOCK_INJECTION_H
+
+#ifdef SPOCK_RANDOM_DELAYS
+
+extern void spock_random_delay(void);
+#define SPOCK_WORKER_DELAY()	spock_random_delay()
+
+#elif PG_VERSION_NUM >= 170000
+
+#include "utils/injection_point.h"
+#define SPOCK_WORKER_DELAY()	INJECTION_POINT("spock-worker-delay", NULL)
+
+#else
+
+#define SPOCK_WORKER_DELAY()	((void) 0)
+
+#endif							/* SPOCK_RANDOM_DELAYS / PG_VERSION_NUM */
+
+#endif							/* SPOCK_INJECTION_H */

src/spock_apply.c

@@ -84,6 +84,7 @@
 #include "spock_common.h"
 #include "spock_readonly.h"
 #include "spock.h"
+#include "spock_injection.h"
 
 
 PGDLLEXPORT void spock_apply_main(Datum main_arg);
@@ -509,6 +510,8 @@ handle_begin(StringInfo s)
 	int			sub_name_len = strlen(MySubscription->name);
 	char	   *slot_name;
 
+	SPOCK_WORKER_DELAY();
+
 	/*
 	 * To get here we must have connected successfully and the replication
 	 * stream is delivering the first transaction. At this point we switch to

src/spock_injection.c

@@ -0,0 +1,46 @@
+/*-------------------------------------------------------------------------
+ *
+ * spock_injection.c
+ *		Unconditional random delay for Spock worker start/finish sites.
+ *
+ * spock_random_delay() is compiled in only when SPOCK_RANDOM_DELAYS is set
+ * in the environment at build time (make SPOCK_RANDOM_DELAYS=1).  It sleeps
+ * for a random duration in [1, SPOCK_INJ_MAX_DELAY_MS] ms.
+ *
+ * On PG17+ without SPOCK_RANDOM_DELAYS, SPOCK_WORKER_DELAY() expands to
+ * INJECTION_POINT("spock-worker-delay") instead -- attach a callback via
+ * the core injection_points module when needed.
+ *
+ * Copyright (c) 2022-2026, pgEdge, Inc.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#ifdef SPOCK_RANDOM_DELAYS
+
+#include "common/pg_prng.h"
+#include "port.h"
+
+/* Maximum random delay, in milliseconds. */
+#define SPOCK_INJ_MAX_DELAY_MS	100
+
+/*
+ * spock_random_delay
+ *		Sleep for a random duration in [1, SPOCK_INJ_MAX_DELAY_MS] ms.
+ *
+ * Uses the global PostgreSQL PRNG so the sequence is reproducible when the
+ * seed is fixed, which helps in deterministic test scenarios.
+ */
+void
+spock_random_delay(void)
+{
+	long	delay_ms = 1 + (long) (pg_prng_uint64(&pg_global_prng_state) %
+								   SPOCK_INJ_MAX_DELAY_MS);
+
+	elog(LOG, "Spock random delay: sleeping %ld ms", delay_ms);
+	pg_usleep(delay_ms * 1000L);
+}
+
+#endif							/* SPOCK_RANDOM_DELAYS */

src/spock_worker.c

@@ -51,6 +51,7 @@
 #include "spock_exception_handler.h"
 #include "spock_group.h"
 #include "spock_shmem.h"
+#include "spock_injection.h"
 
 typedef struct signal_worker_item
 {
@@ -341,6 +342,13 @@ spock_worker_attach(int slot, SpockWorkerType type)
 	/* Now safe to process signals */
 	BackgroundWorkerUnblockSignals();
 
+	/*
+	 * Allow tests to delay worker startup, e.g. to exercise race conditions
+	 * in the manager's restart-delay logic.  Fires before the worker
+	 * announces itself in shared memory.
+	 */
+	SPOCK_WORKER_DELAY();
+
 	MyProcPort = (Port *) calloc(1, sizeof(Port));
 
 	LWLockAcquire(SpockCtx->lock, LW_EXCLUSIVE);
@@ -407,6 +415,13 @@ spock_worker_detach(bool crash)
 	if (MySpockWorker == NULL)
 		return;
 
+	/*
+	 * Allow tests to observe the worker just before it releases its shared
+	 * memory slot.  The worker type and proc pointer are still valid here,
+	 * so callers can distinguish worker types if needed.
+	 */
+	SPOCK_WORKER_DELAY();
+
 	LWLockAcquire(SpockCtx->lock, LW_EXCLUSIVE);
 
 	Assert(MySpockWorker->proc == MyProc);

tests/docker/Dockerfile-step-1.el9

@@ -50,13 +50,18 @@ ARG DBPASSWD=testpass
 ARG DBNAME=demo
 ARG DBPORT=5432
 ARG MAKE_JOBS=4
+# When set to any non-empty value, Spock is compiled with -DSPOCK_RANDOM_DELAYS,
+# injecting random 1-100 ms delays at every transaction begin, worker start,
+# and worker finish.  Used by the random-delays-regress workflow.
+ARG SPOCK_RANDOM_DELAYS=
 
 # Export as environment variables for build-time and runtime
 ENV PGVER=${PGVER} \
 	DBUSER=${DBUSER} \
 	DBPASSWD=${DBPASSWD} \
 	DBNAME=${DBNAME} \
-	DBPORT=${DBPORT}
+	DBPORT=${DBPORT} \
+	SPOCK_RANDOM_DELAYS=${SPOCK_RANDOM_DELAYS}
 
 # PostgreSQL paths
 ENV PATH="/home/pgedge/pgedge:/home/pgedge/pgedge/pg${PGVER}/bin:${PATH}" \
@@ -210,7 +215,7 @@ RUN set -eux && \
 	echo "Building Spock Extension" && \
 	echo "========================================" && \
 	make clean > /dev/null && \
-	make -j${MAKE_JOBS} > /dev/null && \
+	make -j${MAKE_JOBS} ${SPOCK_RANDOM_DELAYS:+SPOCK_RANDOM_DELAYS=1} > /dev/null && \
 	make install > /dev/null && \
 	echo "export SPOCK_SOURCE_DIR=/home/pgedge/spock" >> /home/pgedge/.bashrc && \
 	echo "Spock installation complete"

tests/regress/expected/att_list.out

@@ -312,3 +312,13 @@ NOTICE:  drop cascades to table basic_dml membership in replication set default
  t
 (1 row)
 
+-- Sync subscription here explicitly: in case following test utilises this
+-- table, subscriber will have guarantees that this DROP has been applied.
+SELECT spock.sync_event() AS sync_lsn \gset
+\c :subscriber_dsn
+CALL spock.wait_for_sync_event(NULL, 'test_provider', :'sync_lsn', 60);
+ result 
+--------
+ t
+(1 row)
+

tests/regress/expected/autoddl.out

@@ -131,8 +131,15 @@ SELECT objname, label FROM pg_seclabels;
  slabel1.x | spock.delta_apply
 (1 row)
 
--- Short round trip to check that subscriber has the security label
+-- Wait for the apply worker to process the security label before checking.
+SELECT spock.sync_event() AS sync_lsn \gset
 \c :subscriber_dsn
+CALL spock.wait_for_sync_event(NULL, 'test_provider', :'sync_lsn', 60);
+ result 
+--------
+ t
+(1 row)
+
 SELECT objname, label FROM pg_seclabels;
   objname  |       label       
 -----------+-------------------
@@ -148,8 +155,15 @@ INFO:  DDL statement replicated.
  t
 (1 row)
 
--- Short round trip to check that subscriber has removed the security label too
+-- Wait for the apply worker to process the removal before checking.
+SELECT spock.sync_event() AS sync_lsn \gset
 \c :subscriber_dsn
+CALL spock.wait_for_sync_event(NULL, 'test_provider', :'sync_lsn', 60);
+ result 
+--------
+ t
+(1 row)
+
 SELECT objname, label FROM pg_seclabels;
  objname | label 
 ---------+-------

tests/regress/expected/generated_columns.out

@@ -189,7 +189,18 @@ SELECT spock.wait_slot_confirm_lsn(NULL, NULL);
 
 -- Add more data before resync
 INSERT INTO gen_resync (id, data) VALUES (3, 3), (4, 4);
+-- Wait for apply worker to catch up before resyncing; without this, the
+-- apply worker may have already inserted some rows and the sync COPY
+-- would fail with a duplicate-key violation.
+SELECT spock.sync_event() AS sync_lsn
+\gset
 \c :subscriber_dsn
+CALL spock.wait_for_sync_event(NULL, 'test_provider', :'sync_lsn', 60);
+ result 
+--------
+ t
+(1 row)
+
 SELECT spock.sub_resync_table('test_subscription', 'gen_resync', true);
  sub_resync_table 
 ------------------

tests/regress/expected/primary_key.out

@@ -219,6 +219,19 @@ SELECT * FROM pk_users ORDER BY id;
 
 \c :provider_dsn
 \set VERBOSITY terse
+-- Ensure all prior DML (including the rows that create the duplicate on the
+-- subscriber) has been applied before issuing the DDL.  Without this, delays
+-- in the apply worker may cause the subscriber to receive CREATE UNIQUE INDEX
+-- before the conflicting row exists, making it succeed instead of fail.
+SELECT spock.sync_event() AS sync_lsn \gset
+\c :subscriber_dsn
+CALL spock.wait_for_sync_event(NULL, 'test_provider', :'sync_lsn', 60);
+ result 
+--------
+ t
+(1 row)
+
+\c :provider_dsn
 SELECT quote_literal(pg_current_wal_lsn()) as curr_lsn
 \gset
 SELECT spock.replicate_ddl($$
@@ -242,12 +255,6 @@ SELECT attname, attnotnull, attisdropped from pg_attribute where attrelid = 'pk_
  address    | f          | f
 (5 rows)
 
-SELECT spock.wait_slot_confirm_lsn(NULL, :curr_lsn);
- wait_slot_confirm_lsn 
------------------------
-
-(1 row)
-
 \c :subscriber_dsn
 SELECT attname, attnotnull, attisdropped from pg_attribute where attrelid = 'pk_users'::regclass and attnum > 0 order by attnum;
   attname   | attnotnull | attisdropped 

tests/regress/expected/sync_table.out

@@ -129,7 +129,18 @@ SELECT sum(x), count(*) FROM test_sync;
    0 |    20
 (1 row)
 
+-- Wait for apply worker to catch up before resyncing; without this, the
+-- apply worker may have already inserted some rows and the sync COPY
+-- would fail with a duplicate-key violation.
+SELECT spock.sync_event() AS sync_lsn
+\gset
 \c :subscriber_dsn
+CALL spock.wait_for_sync_event(NULL, 'test_provider', :'sync_lsn', 60);
+ result 
+--------
+ t
+(1 row)
+
 -- Restart syncing this specific table, wait until the process finish and check
 -- all the data stay consistent
 SELECT sync_kind,sub_name,sync_nspname,sync_relname,sync_status, sync_statuslsn <> '0/0'

tests/regress/sql/att_list.sql

@@ -136,3 +136,9 @@ SELECT * FROM basic_dml ORDER BY id;
 SELECT spock.replicate_ddl($$
 	DROP TABLE public.basic_dml CASCADE;
 $$);
+
+-- Sync subscription here explicitly: in case following test utilises this
+-- table, subscriber will have guarantees that this DROP has been applied.
+SELECT spock.sync_event() AS sync_lsn \gset
+\c :subscriber_dsn
+CALL spock.wait_for_sync_event(NULL, 'test_provider', :'sync_lsn', 60);