Reduce CI retry amplification and harden relay-flaky tests

Author: tripledoublev

.github/workflows/lint_and_test.yml

@@ -32,7 +32,6 @@ jobs:
       - name: Run tests with retries
         env:
           RUST_MIN_STACK: 8388608
-        # Retries configured in nextest.toml (no CLI override - P2P tests get 15 retries)
+        # Retries configured in nextest.toml (no CLI override)
         # --no-fail-fast: run all tests even when some fail, so we see full results
         run: cargo nextest run --test-threads=1 --no-fail-fast
-

nextest.toml

@@ -14,7 +14,13 @@ test-threads = 1
 # Mark tests slow after 60s (informational)
 slow-timeout = "60s"
 
-# P2P tests need many retries for flaky peer discovery - keep retrying until they pass
+# P2P tests are flaky in CI, but very high retry counts can stretch failures to >1h.
+# Keep retries moderate so truly broken runs fail fast.
 [[profile.default.overrides]]
-filter = 'test(test_replicate_group) | test(test_refresh_joined_group) | test(test_join_group)'
-retries = { backoff = "exponential", count = 15, delay = "15s", max-delay = "90s", jitter = true }
+filter = 'test(test_replicate_group) | test(test_join_group)'
+retries = { backoff = "exponential", count = 4, delay = "8s", max-delay = "45s", jitter = true }
+
+[[profile.default.overrides]]
+filter = 'test(test_refresh_joined_group)'
+retries = { backoff = "exponential", count = 2, delay = "8s", max-delay = "30s", jitter = true }
+test-timeout = "300s"

src/lib.rs

@@ -198,13 +198,46 @@ mod tests {
 
         assert_eq!(resp.groups.len(), 0);
 
-        let req = test::TestRequest::post()
-            .uri("/api/groups")
-            .set_json(RequestName {
-                name: "example".to_string(),
-            })
-            .to_request();
-        let group: SnowbirdGroup = test::call_and_read_body_json(&app, req).await;
+        // Group creation can fail transiently in CI when relay discovery is unstable.
+        // Retry only this step so basic_test is robust but still fails fast on real errors.
+        let mut group_opt: Option<SnowbirdGroup> = None;
+        let mut last_create_group_error = String::new();
+        for attempt in 1..=6 {
+            let req = test::TestRequest::post()
+                .uri("/api/groups")
+                .set_json(RequestName {
+                    name: "example".to_string(),
+                })
+                .to_request();
+            let resp = test::call_service(&app, req).await;
+            let resp_status = resp.status();
+            let body = test::read_body(resp).await;
+
+            if resp_status.is_success() {
+                match serde_json::from_slice::<SnowbirdGroup>(&body) {
+                    Ok(group) => {
+                        group_opt = Some(group);
+                        break;
+                    }
+                    Err(e) => {
+                        last_create_group_error = format!("invalid success payload: {e}; body={body:?}");
+                    }
+                }
+            } else {
+                let body_text = String::from_utf8_lossy(&body).to_string();
+                last_create_group_error = format!("status={resp_status}, body={body_text}");
+                if !body_text.contains("couldn't look up relay") {
+                    break;
+                }
+            }
+
+            if attempt < 6 {
+                tokio::time::sleep(Duration::from_secs(3)).await;
+            }
+        }
+        let group = group_opt.expect(&format!(
+            "Creating group failed after retries: {last_create_group_error}"
+        ));
 
         assert_eq!(group.name, Some("example".to_string()));
 
@@ -920,7 +953,7 @@ mod tests {
         // Retry refresh until P2P replication converges: the refresh must succeed AND
         // the response must contain the creator's repo (by name) with the uploaded file.
         // Repo names and file lists propagate via DHT and may lag behind the initial join.
-        let mut refresh_retries = 30;
+        let mut refresh_retries = 12;
         let refresh_data: serde_json::Value = loop {
             let refresh_req = test::TestRequest::post()
                 .uri(&format!("/api/groups/{}/refresh", group.id()))
@@ -940,20 +973,20 @@ mod tests {
                     break data;
                 }
                 log::warn!("Refresh succeeded but repo data not yet propagated (attempt {})",
-                    30 - refresh_retries + 1);
+                    12 - refresh_retries + 1);
             } else {
                 log::warn!("Refresh failed (attempt {}): status={}, body={:?}",
-                    30 - refresh_retries + 1,
+                    12 - refresh_retries + 1,
                     resp.status(),
                     test::read_body(resp).await
                 );
             }
 
             refresh_retries -= 1;
             if refresh_retries == 0 {
-                panic!("Refresh did not converge after 30 attempts.");
+                panic!("Refresh did not converge after 12 attempts.");
             }
-            tokio::time::sleep(Duration::from_secs(5)).await;
+            tokio::time::sleep(Duration::from_secs(4)).await;
         };
 
         assert_eq!(refresh_data["status"], "success", "First refresh status should be success");