Restore analytics raw snapshot; pause fact backfill

docs/analytics.md

@@ -109,15 +109,20 @@ failure is logged and deferred to the dirty reconciler rather than failing the
 beacon. A compare-and-replace check rejects a raw snapshot if another event
 advanced the revision on either side of its read, preventing an older concurrent
 rebuild from winning last. Deletes are ignored, so a later raw-retention policy
-cannot subtract retained facts. A background worker with a 30-second database
-lease and persisted `(occurred_at, UUID)` cursor backfills in small batches with
-exponential backoff on datastore errors. It runs after database initialization,
-advances through scan and final reconciliation phases, and keeps processing
-dirty keys after readiness; it never blocks a data-backed route from opening.
-
-Until reconciliation completes, each render performs the legacy three-second
-raw snapshot. The first request for each of the four windows then compares the
-fact and legacy dashboards structurally and persists a four-bit parity mask.
+cannot subtract retained facts. The background worker is **opt-in** via
+`ANALYTICS_FACTS_BACKFILL=1`: with a 30-second database lease and persisted
+`(occurred_at, UUID)` cursor it backfills in small batches with exponential
+backoff on datastore errors. It starts after database initialization when
+enabled, advances through scan and final reconciliation phases, and keeps
+processing dirty keys after readiness; it never blocks a data-backed route from
+opening.
+
+Until reconciliation completes (or while the worker is idle), each render
+performs the legacy raw snapshot: one bounded events query plus a prior-session
+probe over only the idle window before cutoff, with the window ∩ prior
+intersection done in Rust. The first request for each of the four windows then
+compares the fact and legacy dashboards structurally and persists a four-bit
+parity mask.
 Only mask 15 activates fact-only reads. Any fact query or decode failure falls
 back to the raw snapshot. Fact loads include the requested UTC days and the
 preceding UTC day; only pageviews in the exact prior 30-minute slice contribute

docs/railway-deploy.md

@@ -82,6 +82,14 @@ Flag-on also needs the `db.` Tunnel hostname
 ([cloudflare-deploy.md](cloudflare-deploy.md)); unsetting the variables
 removes the access method again at the next boot.
 
+Analytics visitor-day backfill ([analytics.md](analytics.md)) is similarly
+opt-in on the web service — leave it unset so the legacy raw dashboard stays
+healthy; set it only when deliberately finishing the fact rollout:
+
+```text
+ANALYTICS_FACTS_BACKFILL=1   # leased scan/reconcile worker; off when unset
+```
+
 `HOST=0.0.0.0` is baked into the web image; Railway injects `PORT`. Pin it to
 `8080` so the Tunnel origin stays stable.
 

docs/surrealdb-notes.md

@@ -51,10 +51,11 @@ so an epoch-1 rollback can continue using its old raw analytics loader. Diary
 epochs remain strictly version-fenced and do not share this rule.
 
 Analytics epoch 2 owns `analytics_visitor_days`, its rebuild function/event,
-and the leased backfill cursor. The background task starts only after bootstrap
-returns; while it is scanning or reconciling, request-path rebuilds stay off so
-live writes only dirty keys. Facts do not become the dashboard source until all
-four supported windows have exact legacy parity. See `docs/analytics.md`.
+and the leased backfill cursor. The background task is opt-in
+(`ANALYTICS_FACTS_BACKFILL=1`) and starts only after bootstrap returns; while it
+is scanning or reconciling, request-path rebuilds stay off so live writes only
+dirty keys. Facts do not become the dashboard source until all four supported
+windows have exact legacy parity. See `docs/analytics.md`.
 
 The diary is the exception to pure reconciliation because offline clients need
 an exact activation boundary. `src/data/diary_migrations.rs` applies its

src/app/analytics/dashboard.rs

@@ -10,8 +10,6 @@ use benjisponge::data::{
     Db, analytics_facts,
     analytics_models::{AnalyticsEvent, AnalyticsVisitorDay},
 };
-use serde::Deserialize;
-use surrealdb::types::SurrealValue;
 use tokio::time::timeout;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -155,12 +153,6 @@ pub struct Campaign {
     pub visitors: i64,
 }
 
-#[derive(Deserialize, SurrealValue)]
-struct SnapshotRows {
-    events: Vec<AnalyticsEvent>,
-    prior_sessions: Vec<String>,
-}
-
 pub async fn load(db: &Db, cutoff: i64) -> anyhow::Result<Dashboard> {
     let current = current_timestamp()?;
     match analytics_facts::load(db, cutoff).await {
@@ -208,57 +200,62 @@ pub async fn load(db: &Db, cutoff: i64) -> anyhow::Result<Dashboard> {
 }
 
 async fn load_raw(db: &Db, cutoff: i64, current: i64) -> anyhow::Result<Dashboard> {
-    // `prior_sessions` only needs the idle window before `$cutoff`. A session
-    // id that survives across the boundary must have had activity inside that
-    // window (see `SESSION_IDLE_SECONDS`); scanning unbounded history made the
-    // nested IN pathologically slow on short ranges once older traffic existed
-    // alongside a busy in-window set (the 7d standby card).
-    const SNAPSHOT: &str = "
-        RETURN {
-            events: (
-                SELECT *, record::id(id) AS id
-                FROM analytics_events
-                WHERE occurred_at >= $cutoff
-            ),
-            prior_sessions: (
-                SELECT VALUE session_id
-                FROM analytics_events
-                WHERE kind = 'pageview'
-                    AND occurred_at < $cutoff
-                    AND occurred_at >= $prior_floor
-                    AND session_id IN (
-                        SELECT VALUE session_id
-                        FROM analytics_events
-                        WHERE kind = 'pageview'
-                            AND occurred_at >= $cutoff
-                    )
-                GROUP BY session_id
-            )
-        }";
-
+    // Prior markers only need the idle window before `$cutoff`. Intersect that
+    // tiny candidate set with in-window pageview sessions in Rust — a nested
+    // `session_id IN (SELECT … from the whole window)` re-scanned the busy
+    // range under every dashboard load and timed out once fact backfill was
+    // also contending for SurrealDB.
     let prior_floor = cutoff.saturating_sub(super::db::SESSION_IDLE_SECONDS);
-    let snapshot = timeout(Duration::from_secs(3), async {
+    let (events, prior_sessions) = timeout(Duration::from_secs(8), async {
         let mut response = db
-            .query(SNAPSHOT)
+            .query(
+                "SELECT *, record::id(id) AS id
+                 FROM analytics_events
+                 WHERE occurred_at >= $cutoff",
+            )
             .bind(("cutoff", cutoff))
-            .bind(("prior_floor", prior_floor))
             .await
             .context("analytics snapshot query failed")?
             .check()
             .context("analytics snapshot query failed")?;
-        let snapshot: Option<SnapshotRows> = response
+        let events: Vec<AnalyticsEvent> = response
             .take(0)
             .context("analytics snapshot decoding failed")?;
-        snapshot.context("analytics snapshot query returned no rows")
+
+        let mut response = db
+            .query(
+                "SELECT VALUE session_id
+                 FROM analytics_events
+                 WHERE kind = 'pageview'
+                     AND occurred_at < $cutoff
+                     AND occurred_at >= $prior_floor
+                 GROUP BY session_id",
+            )
+            .bind(("cutoff", cutoff))
+            .bind(("prior_floor", prior_floor))
+            .await
+            .context("analytics prior-session query failed")?
+            .check()
+            .context("analytics prior-session query failed")?;
+        let candidates: Vec<String> = response
+            .take(0)
+            .context("analytics prior-session decoding failed")?;
+
+        let window_sessions: HashSet<&str> = events
+            .iter()
+            .filter(|event| event.kind == "pageview")
+            .map(|event| event.session_id.as_str())
+            .collect();
+        let prior_sessions = candidates
+            .into_iter()
+            .filter(|session_id| window_sessions.contains(session_id.as_str()))
+            .collect::<HashSet<_>>();
+        Ok::<_, anyhow::Error>((events, prior_sessions))
     })
     .await
-    .context("analytics snapshot exceeded three seconds")??;
-    Ok(aggregate(
-        &snapshot.events,
-        &snapshot.prior_sessions.into_iter().collect(),
-        cutoff,
-        current,
-    ))
+    .context("analytics snapshot exceeded eight seconds")??;
+
+    Ok(aggregate(&events, &prior_sessions, cutoff, current))
 }
 
 fn current_timestamp() -> anyhow::Result<i64> {

src/data/analytics_facts.rs

@@ -52,6 +52,18 @@ struct BackfillState {
 }
 
 pub fn start_backfill(db: Db) {
+    // Off by default: the leased scan rebuilds enough visitor-days to reset
+    // SurrealDB connections and starve the legacy dashboard. Re-enable with
+    // ANALYTICS_FACTS_BACKFILL=1 once raw reads are healthy again.
+    match std::env::var("ANALYTICS_FACTS_BACKFILL") {
+        Ok(value) if matches!(value.as_str(), "1" | "true" | "TRUE" | "yes") => {}
+        _ => {
+            eprintln!(
+                "analytics facts: backfill worker idle (set ANALYTICS_FACTS_BACKFILL=1 to enable)"
+            );
+            return;
+        }
+    }
     tokio::spawn(async move {
         let owner = Uuid::new_v4().to_string();
         let mut failures: u32 = 0;