Retry 502 Bad Gateway on every API call instead of failing the pipeline

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "corgea"
-version = "1.13.0"
+version = "1.14.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

README.md

@@ -39,14 +39,30 @@ third-party report.
 
 All three exit 1 if the scan fails, printing the reason and the scanners that hit
 problems. A scan that completes with a scanner missing exits 0 with a warning.
-Waiting gives up after 10 hours; override with `CORGEA_SCAN_TIMEOUT_SECONDS`.
+While waiting, the scan's status is read every 3 seconds. Waiting gives up after
+10 hours; override with `CORGEA_SCAN_TIMEOUT_SECONDS`.
 `--fail`/`--block-on` then wait up to 15 minutes for blocking rules to be
 evaluated; override with `CORGEA_BLOCKING_RULES_TIMEOUT_SECONDS`.
 
 `--out-format`/`--out-file` and `--sbom` are honored whether or not a gate
 trips: both are written before `--fail`/`--block-on` are evaluated, so a scan
 that exits 1 on a blocking rule still leaves its report behind to ingest.
 
+### Gateway errors are retried, not surfaced
+
+Every Corgea API call the CLI makes — uploads included — replays itself when the
+platform's proxy answers `502 Bad Gateway`, waiting 10s, then 30s, then 50s. A
+request that is still answered 502 after those three retries fails the command
+in the usual way, so a pipeline exits non-zero on a real outage and rides out the
+blips a busy platform produces under parallel scans. Each retry is logged, and
+the count belongs to a single request: any successful call starts the next one
+with the full three retries again.
+
+The one place this stops early is `corgea upload`'s per-file source upload: a 502
+that outlives the retries there is taken as the platform being unavailable rather
+than one bad file, so the remaining paths are reported as unsent instead of each
+spending another 90 seconds.
+
 ### Skipping a re-scan of the same commit
 
 A pipeline that re-runs on an unchanged commit can reuse the scan it already

src/scan.rs

@@ -328,8 +328,9 @@ pub fn upload_scan(
     let mut uploaded_paths = HashSet::new();
     let mut uploaded_count = 0;
     let mut upload_error_count = 0;
+    let mut gateway_gave_up = false;
 
-    for path in &paths {
+    'files: for path in &paths {
         if !Path::new(&path).exists() {
             log::error!(
                 "Required file {} not found which is required for the scan, exiting.",
@@ -354,7 +355,7 @@ pub fn upload_scan(
 
         while attempts < 3 && !success {
             debug(&format!("POST: {}", src_upload_url));
-            let res = utils::api::retry_on_network_error("file upload", || {
+            let res = utils::api::send_with_retries("a source file upload", || {
                 let form = reqwest::blocking::multipart::Form::new()
                     .file("file", fp)
                     .expect("Failed to read file");
@@ -372,6 +373,20 @@ pub fn upload_scan(
                             "Code upload failed with status: {}. Response body: {}",
                             status, body
                         ));
+                        // A 502 that outlived the retry schedule is the platform
+                        // being unavailable, not this one file. Retrying it here
+                        // would spend the schedule twice over, and walking the
+                        // remaining paths would spend a fresh 90 seconds on each
+                        // of them, so stop uploading source files altogether.
+                        if utils::api::is_gateway_error(status) {
+                            log::warn!(
+                                "Failed to upload file {} after the gateway retries: {}",
+                                path,
+                                status
+                            );
+                            gateway_gave_up = true;
+                            break 'files;
+                        }
                         log::warn!("Failed to upload file {} {}... retrying", status, path);
                         std::thread::sleep(std::time::Duration::from_secs(1));
                         attempts += 1;
@@ -402,6 +417,19 @@ pub fn upload_scan(
         }
     }
 
+    // Everything the aborted walk never attempted still counts as unsent, or
+    // the closing summary would report one failure for a whole skipped tree.
+    if gateway_gave_up {
+        let distinct: HashSet<&String> = paths.iter().collect();
+        let unsent = distinct.len() - uploaded_paths.len();
+        upload_error_count += unsent;
+        log::warn!(
+            "Stopped uploading source files: Corgea was still answering 502 after the retries. {} of {} files were not sent.",
+            unsent,
+            distinct.len()
+        );
+    }
+
     if uploaded_count == 0 {
         log::error!("Failed to upload any files for the scan, exiting.");
         std::process::exit(1);
@@ -440,7 +468,7 @@ pub fn upload_scan(
                 index + 1,
                 total_chunks
             ));
-            let response = utils::api::retry_on_network_error("scan chunk upload", || {
+            let response = utils::api::send_with_retries("a scan report chunk upload", || {
                 client
                     .post(&scan_upload_url)
                     .header(header::CONTENT_TYPE, "application/json")
@@ -491,7 +519,7 @@ pub fn upload_scan(
         last_response.expect("Failed to upload scan.")
     } else {
         debug(&format!("POST: {}", scan_upload_url));
-        utils::api::retry_on_network_error("scan upload", || {
+        utils::api::send_with_retries("the scan report upload", || {
             client
                 .post(&scan_upload_url)
                 .header(header::CONTENT_TYPE, "application/json")
@@ -572,7 +600,7 @@ pub fn upload_scan(
     if git_config_path.exists() {
         debug("Uploading .git/config");
         debug(&format!("POST: {}", git_config_upload_url));
-        let res = utils::api::retry_on_network_error("git config upload", || {
+        let res = utils::api::send_with_retries("the git config upload", || {
             let form = reqwest::blocking::multipart::Form::new()
                 .file("file", git_config_path)
                 .expect("Failed to read file");

src/scanners/blast.rs

@@ -16,6 +16,15 @@ use std::time::{Duration, Instant};
 const SCAN_TIMEOUT_ENV: &str = "CORGEA_SCAN_TIMEOUT_SECONDS";
 const DEFAULT_SCAN_TIMEOUT: Duration = Duration::from_secs(10 * 60 * 60);
 
+/// How long to pause between scan status reads.
+///
+/// Scans run for minutes, so reading more often than this only adds load: at
+/// one read a second, a single wait against the default 10-hour budget can
+/// reach 36,000 requests, and a pipeline scanning in parallel multiplies that
+/// by every concurrent wait. Three seconds costs a pipeline no more than a few
+/// seconds of extra latency on the final status.
+const SCAN_POLL_INTERVAL: Duration = Duration::from_secs(3);
+
 /// Overrides how long the CI gate waits for blocking rules to be evaluated.
 const BLOCKING_RULES_TIMEOUT_ENV: &str = "CORGEA_BLOCKING_RULES_TIMEOUT_SECONDS";
 
@@ -1034,7 +1043,7 @@ pub fn wait_for_scan(config: &Config, scan_id: &str, budget: WaitBudget) {
     let mut last_status = String::from("unknown");
 
     let result = loop {
-        thread::sleep(Duration::from_secs(1));
+        thread::sleep(SCAN_POLL_INTERVAL);
         // Every read is capped to what is left of the budget: on the client's
         // own timeout a stalled read would otherwise keep us going long past
         // the wait the user asked for.
@@ -1781,6 +1790,18 @@ mod tests {
         assert_eq!(format_timeout(Duration::from_secs(90)), "90s");
     }
 
+    #[test]
+    fn poll_interval_is_the_documented_one_and_short_against_the_budget() {
+        assert_eq!(SCAN_POLL_INTERVAL, Duration::from_secs(3));
+        // A pause that ever grew toward the budget would spend the whole wait
+        // asleep and report a finished scan long after it finished.
+        assert!(
+            SCAN_POLL_INTERVAL < DEFAULT_SCAN_TIMEOUT / 100,
+            "poll interval {SCAN_POLL_INTERVAL:?} is not short against the \
+             {DEFAULT_SCAN_TIMEOUT:?} budget it runs inside"
+        );
+    }
+
     #[test]
     fn blocking_rules_timeout_outlasts_the_doghouse_wait_windows() {
         // This side fails closed on its own deadline, so it must never expire