From b681b3d1b3223e38a7c58be066307c7ba95de8bc Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Sun, 10 May 2026 22:52:40 +0000 Subject: [PATCH] fix(integration): Fix flaky TestParquetFuzz Two issues caused this test to be flaky: 1. Cortex started before the block was uploaded to minio, causing the compactor to see a partial block on its first scan and skip it. The bucket index was never updated within the 30s poll timeout. Fix: upload the block before starting cortex. 2. The fuzz test compared Cortex (with parquet queryable) against standalone Prometheus without skipping queries with known cross-version semantic differences (stdvar/stddev changed in prometheus/prometheus#14941). The random seed meant some runs would generate these queries and fail. Fix: pass skipStdAggregations=true since this test compares against a standalone Prometheus instance. Also increase the poll timeout from 30s to 60s as a safety margin for slow arm64 CI runners. Signed-off-by: Ben Ye --- integration/parquet_querier_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/integration/parquet_querier_test.go b/integration/parquet_querier_test.go index 2c3d8b9256b..ed8ac4463b0 100644 --- a/integration/parquet_querier_test.go +++ b/integration/parquet_querier_test.go @@ -109,18 +109,20 @@ func TestParquetFuzz(t *testing.T) { minio := e2edb.NewMinio(9000, flags["-blocks-storage.s3.bucket-name"]) require.NoError(t, s.StartAndWaitReady(minio)) - cortex := e2ecortex.NewSingleBinary("cortex", flags, "") - require.NoError(t, s.StartAndWaitReady(cortex)) - storage, err := e2ecortex.NewS3ClientForMinio(minio, flags["-blocks-storage.s3.bucket-name"]) require.NoError(t, err) bkt := bucket.NewUserBucketClient("user-1", storage.GetBucket(), nil) + // Upload the block before starting cortex so the first compactor scan finds + // the complete block and includes it in the bucket index immediately. err = block.Upload(ctx, log.Logger, bkt, filepath.Join(dir, id.String()), metadata.NoneFunc) require.NoError(t, err) + cortex := e2ecortex.NewSingleBinary("cortex", flags, "") + require.NoError(t, s.StartAndWaitReady(cortex)) + // Wait until we convert the blocks - cortex_testutil.Poll(t, 30*time.Second, true, func() interface{} { + cortex_testutil.Poll(t, 60*time.Second, true, func() interface{} { found := false foundBucketIndex := false @@ -173,7 +175,7 @@ func TestParquetFuzz(t *testing.T) { } ps := promqlsmith.New(rnd, lbls, opts...) - runQueryFuzzTestCases(t, ps, c1, c2, end, start, end, scrapeInterval, 1000, false) + runQueryFuzzTestCases(t, ps, c1, c2, end, start, end, scrapeInterval, 1000, true) require.NoError(t, cortex.WaitSumMetricsWithOptions(e2e.Greater(0), []string{"cortex_parquet_queryable_blocks_queried_total"}, e2e.WithLabelMatchers( labels.MustNewMatcher(labels.MatchEqual, "type", "parquet"))))