From af22b3e60d418f50de534bb3b24f98cb107dd223 Mon Sep 17 00:00:00 2001 From: Falcon Ortiz Date: Tue, 8 Sep 2026 15:39:18 +0200 Subject: [PATCH] SpaceXAI pack: PR-A/D/F1/F5 ControlledSourceReexposure (default off) Creator reach equilibrium proposal from @falconortiz / IRE. - PR-A: FS A/B arms documented; defaults stay Follow=4 Profile=0 CopyLink=20 - PR-F1: EnablePreferOriginalRetweetDedup (default false) - PR-F5: ControlledSourceReexposure Enable=false K=1 InNetworkOnly; source_key RT|quote|id - PR-D: serve card classify hooks (original|rt|quote) SpaceXAI owns merge. Evidence + eng notes under docs/spacexai-ire-2026-09-08/. --- docs/spacexai-ire-2026-09-08/ENG_GREEN.md | 24 ++ docs/spacexai-ire-2026-09-08/PATCHES.md | 43 +++ docs/spacexai-ire-2026-09-08/RESULTS.md | 25 ++ docs/spacexai-ire-2026-09-08/SEND_GO.md | 5 + .../SPACEXAI_COVER_MIXED_FINAL.md | 33 +++ .../SPACEXAI_F5_IMPLEMENTATION.md | 43 +++ .../SPACEXAI_MIXED_PR_F5.md | 109 +++++++ .../SPACEXAI_PARAM_SKETCH_PR_A.md | 135 +++++++++ .../SPACEXAI_PR_D_TELEMETRY_NOTE.md | 30 ++ .../SPACEXAI_RT_QUOTE_REQUEST.md | 52 ++++ docs/spacexai-ire-2026-09-08/STATUS.md | 64 ++++ .../phoenix_candidate_pipeline.rs | 3 + .../controlled_source_reexposure_filter.rs | 280 ++++++++++++++++++ home-mixer/filters/mod.rs | 1 + .../filters/previously_served_posts_filter.rs | 17 +- .../filters/retweet_deduplication_filter.rs | 134 ++++++++- home-mixer/params/param.rs | 48 +++ home-mixer/util/candidates_util.rs | 36 +++ home-mixer/util/mod.rs | 1 + home-mixer/util/serve_card_classify.rs | 79 +++++ 20 files changed, 1147 insertions(+), 15 deletions(-) create mode 100644 docs/spacexai-ire-2026-09-08/ENG_GREEN.md create mode 100644 docs/spacexai-ire-2026-09-08/PATCHES.md create mode 100644 docs/spacexai-ire-2026-09-08/RESULTS.md create mode 100644 docs/spacexai-ire-2026-09-08/SEND_GO.md create mode 100644 docs/spacexai-ire-2026-09-08/SPACEXAI_COVER_MIXED_FINAL.md create mode 100644 docs/spacexai-ire-2026-09-08/SPACEXAI_F5_IMPLEMENTATION.md create mode 100644 docs/spacexai-ire-2026-09-08/SPACEXAI_MIXED_PR_F5.md create mode 100644 docs/spacexai-ire-2026-09-08/SPACEXAI_PARAM_SKETCH_PR_A.md create mode 100644 docs/spacexai-ire-2026-09-08/SPACEXAI_PR_D_TELEMETRY_NOTE.md create mode 100644 docs/spacexai-ire-2026-09-08/SPACEXAI_RT_QUOTE_REQUEST.md create mode 100644 docs/spacexai-ire-2026-09-08/STATUS.md create mode 100644 home-mixer/filters/controlled_source_reexposure_filter.rs create mode 100644 home-mixer/util/serve_card_classify.rs diff --git a/docs/spacexai-ire-2026-09-08/ENG_GREEN.md b/docs/spacexai-ire-2026-09-08/ENG_GREEN.md new file mode 100644 index 00000000..c1926ed2 --- /dev/null +++ b/docs/spacexai-ire-2026-09-08/ENG_GREEN.md @@ -0,0 +1,24 @@ +# ENG GREEN — SpaceXAI merge-ready pack +**Reviewer:** Head of Engineering (IRE) · **2026-09-08** +**Base:** `902a06f` · **Patch:** `0001-spacexai-f1-f5-a-d.patch` +**External send:** still **HOLD** until Falcon go + +## Verdict +**ENG GREEN** to land / propose. Acceptance criteria met. + +| # | Criterion | Result | +|---|-----------|--------| +| 1 | PR-A FS only; defaults 4/0/20 | PASS — comment + T1–T3 arms; defaults unchanged | +| 2 | PR-F1 prefer-original FS-gated default false | PASS — first-wins off; Following parity on | +| 3 | PR-F5 Enable=false / K=1; IN-only; source_key RT→quoted→tweet_id; ≤1/slate; after PreviouslyServed; no Age/OON/VF | PASS | +| 4 | PR-D classify hooks only | PASS — `serve_card_classify.rs` | +| 5 | Out list untouched | PASS | +| 6 | Tests green + EXP± | PASS — standalone **22/22** (re-run by HoE); Python smoke **22/22** | + +## Non-blockers (treatment / monorepo follow-ups) +1. **F5 v1 scopes to PreviouslyServed only** — `PreviouslySeen*` still drops impressed related ids. Correct blast radius for landable default-off; treatment arms may need a measured seen-policy later (not open F2b). +2. **D hooks fire on F5 keep path** — full serve-mix baseline when F5 off still needs product Under-the-Hood / monorepo counter wire. +3. **OSS cannot run full home-mixer cargo** — private `xai_*`; standalone crate is the required green bar here. + +## Go +Falcon go → send from GitHub `falconortiz` with patch + mixed cover. Until then: HOLD. diff --git a/docs/spacexai-ire-2026-09-08/PATCHES.md b/docs/spacexai-ire-2026-09-08/PATCHES.md new file mode 100644 index 00000000..8c9e7b80 --- /dev/null +++ b/docs/spacexai-ire-2026-09-08/PATCHES.md @@ -0,0 +1,43 @@ +# Touched files — SpaceXAI pack @ 902a06f + +## Modified + +| File | Pack | +|------|------| +| `home-mixer/params/param.rs` | A (comments), F1 params, F5 params | +| `home-mixer/filters/retweet_deduplication_filter.rs` | F1 | +| `home-mixer/filters/previously_served_posts_filter.rs` | F5 (defer when armed) | +| `home-mixer/filters/mod.rs` | F5 module export | +| `home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs` | F5 wire | +| `home-mixer/util/candidates_util.rs` | F5 `source_key` | +| `home-mixer/util/mod.rs` | D module export | + +## Added + +| File | Pack | +|------|------| +| `home-mixer/filters/controlled_source_reexposure_filter.rs` | F5 | +| `home-mixer/util/serve_card_classify.rs` | D | + +## Standalone test crate (cache, not in x-algorithm) + +| Path | Role | +|------|------| +| `x-algo-cache/merge-ready/spacexai-pack-tests/` | Pure F1/F5/D `cargo test` | +| `x-algo-cache/merge-ready/STATUS.md` | This status | +| `x-algo-cache/merge-ready/PATCHES.md` | This list | +| `x-algo-cache/merge-ready/0001-spacexai-f1-f5-a-d.patch` | `git diff` export | + +## Params added (defaults = today's behavior) + +- `EnablePreferOriginalRetweetDedup`: bool = **false** +- `EnableControlledSourceReexposure`: bool = **false** +- `ControlledSourceReexposureMaxServesK`: u32 = **1** +- `ControlledSourceReexposureMinRequestGap`: u32 = **0** +- `ControlledSourceReexposureInNetworkOnly`: bool = **true** + +## Params NOT changed (PR-A) + +- `FollowAuthorWeight` = 4.0 +- `ProfileClickWeight` = 0.0 +- `ShareViaCopyLinkWeight` = 20.0 diff --git a/docs/spacexai-ire-2026-09-08/RESULTS.md b/docs/spacexai-ire-2026-09-08/RESULTS.md new file mode 100644 index 00000000..77347fe8 --- /dev/null +++ b/docs/spacexai-ire-2026-09-08/RESULTS.md @@ -0,0 +1,25 @@ +# e2e_smoke_f5 RESULTS — 2026-09-08 +**Command:** `python3 smoke_controlled_reexposure.py` +**Outcome:** **22 PASS / 0 FAIL** + +## EXP+ (niche re-exposure) +| Check | Result | +|-------|--------| +| Current: first serve original | PASS | +| Current: 2nd request burned (related/source) | PASS | +| Current: same-slate 3 RTs → 1 | PASS | +| F5: 2nd/3rd IN nudge within K=3 | PASS | +| F5: 4th blocked at K | PASS | +| Attribution = original source_id | PASS | + +## EXP− (snowball) +| Check | Result | +|-------|--------| +| Current 20 RTs → 1 | PASS | +| Current 20 quotes survive RT-dedup | PASS (risk → need quote cap in F5) | +| F5 ≤1/slate + K=3 across requests | PASS | +| F5 rejects OON RT path | PASS | +| Following prefer-native / For You RT-first gap (F1) | PASS | + +## Viability +F5 **not in tree** — implement as FS A/B on top of existing filters. Building blocks all present @ `902a06f`. diff --git a/docs/spacexai-ire-2026-09-08/SEND_GO.md b/docs/spacexai-ire-2026-09-08/SEND_GO.md new file mode 100644 index 00000000..029ba995 --- /dev/null +++ b/docs/spacexai-ire-2026-09-08/SEND_GO.md @@ -0,0 +1,5 @@ +# SEND GO +**Falcon GO:** 2026-09-08 Europe/Madrid +**Eng:** GREEN +**Account:** falconortiz +**Base:** 902a06f diff --git a/docs/spacexai-ire-2026-09-08/SPACEXAI_COVER_MIXED_FINAL.md b/docs/spacexai-ire-2026-09-08/SPACEXAI_COVER_MIXED_FINAL.md new file mode 100644 index 00000000..f4e7a0a5 --- /dev/null +++ b/docs/spacexai-ire-2026-09-08/SPACEXAI_COVER_MIXED_FINAL.md @@ -0,0 +1,33 @@ +# SpaceXAI mixed request — creator reach equilibrium (A + D + F1 + F5) +**From:** Falcon Ortiz (@falconortiz) / IRE · GitHub send: **falconortiz** +**Code:** https://github.com/xai-org/x-algorithm @ `902a06f` +**Pattern:** `docs/BIDIRECTIONAL_BOOST_CHANGE.md` (FS → A/B → ship) +**SpaceXAI owns merge.** IRE proposes only. + +## Plain-language ask +For You maximizes `Σ w·P(action)` with FollowAuthor=4 vs CopyLink=20 and ProfileClick=0. Hard rules (48h age, RT XOR original, source burn, quotes bypass RT-dedup) cut prolonged niche re-exposure. Creator evidence: follow/imp **0.00636%** vs Batch B **0.0644%**; OCR HT **49.1K/500K** at η **3.78%**. + +We request a **mixed program**: +1. **Study/telemetry (PR-D)** — serve mix original vs RT vs quote; re-serve≥2×; negatives if available +2. **Immediate A/Bs** — **PR-A** weight rebalance; **PR-F1** prefer-original same-slate (Following parity) +3. **PR-F5 ControlledSourceReexposure** — implement behind FS **default off** (K=2–3, IN-only, gap, ≤1/slate, **quotes share source cap with RTs**); enable after D baseline + +**Not in this ask:** open burn-off (F2b), post-48h Age rewrite (F4), OCR ranking head (E), VF SpamHighRecall loosen. + +## Evidence (empirical, not SLOs) +| Signal | Value | +|--------|------:| +| Cohort follow/imp | 0.00636% (79/1.24M) | +| Batch B | 0.0644% | +| High-amp vs low-amp strata | 0.0678% vs 0.0053% | +| OCR | 180/500 VF · 49.1K/500K HT | +| F5 smoke | 22/22 PASS (sim @ filters) | + +## Success +↑ follow/imp on non-viral OON; ↑ original serve share after amplify; F5 arms show re-serve 2–3 with no report/mute spike; no quote/RT snowball. + +## Attachments for eng +- `SPACEXAI_PARAM_SKETCH_PR_A.md` +- `SPACEXAI_PR_D_TELEMETRY_NOTE.md` +- `SPACEXAI_F5_IMPLEMENTATION.md` +- `SPACEXAI_MIXED_PR_F5.md` · `e2e_smoke_f5/` diff --git a/docs/spacexai-ire-2026-09-08/SPACEXAI_F5_IMPLEMENTATION.md b/docs/spacexai-ire-2026-09-08/SPACEXAI_F5_IMPLEMENTATION.md new file mode 100644 index 00000000..fdca12de --- /dev/null +++ b/docs/spacexai-ire-2026-09-08/SPACEXAI_F5_IMPLEMENTATION.md @@ -0,0 +1,43 @@ +# PR-F5 — ControlledSourceReexposure implementation note +**Status:** Landable with mixed PR as FS + filter sketch · treatment default **off** +**Depends on:** PR-D counters before enabling K>1 arms +**Companions:** `SPACEXAI_MIXED_PR_F5.md`, `e2e_smoke_f5/` (22/22 PASS sim), HoE quote-cap harden +**Repo:** x-algorithm @ `902a06f` · SpaceXAI owns merge + +## Algorithm (per viewer request) +``` +slate = candidates after hydration +slate = RetweetDedup + prefer-original when F1 on +slate = drop OON RTs (existing OONRetweetReplyFilter — unchanged) +for each candidate: + source = source_key(candidate) + if source in served_or_impressed_related: + if not EnableControlledSourceReexposure: drop # today + elif serves[source] >= K: drop + elif requests_since_last[source] < Gap: drop + elif InNetworkOnly and amplifier not IN: drop + else: keep as re-exposure card + else: + keep # first exposure +# Prefer original over amp when both re-admitted for same source +enforce ≤1 card per source in slate +log serve(source, card_type ∈ {original, rt, quote}) +``` + +## source_key (quotes REQUIRED) +``` +if retweeted_tweet_id: return retweeted_tweet_id +if quoted_tweet_id: return quoted_tweet_id +return tweet_id +``` +RTs and quotes of the same original share one K bucket. Do not ship a false arm on IncludeQuotes. + +## Rollout +1. Land params + logging (D) with **Enable=false** and/or **K=1** (behavior ≡ today) +2. Shadow: compute would-allow rate from serve mix +3. After baseline: A/B **K=2** vs **K=3**, small %; InNetworkOnly=true +4. Guardrails: report / mute / block; duplicate-fatigue proxies +5. Ship winning K or keep off + +## Out of scope +AgeFilter change · OON RT enable · VF / SpamHighRecall loosen · OCR ranking head · open F2b burn-off · F4 post-48h prolong diff --git a/docs/spacexai-ire-2026-09-08/SPACEXAI_MIXED_PR_F5.md b/docs/spacexai-ire-2026-09-08/SPACEXAI_MIXED_PR_F5.md new file mode 100644 index 00000000..62f35d77 --- /dev/null +++ b/docs/spacexai-ire-2026-09-08/SPACEXAI_MIXED_PR_F5.md @@ -0,0 +1,109 @@ +# Mixed SpaceXAI request — study + solutions + implementable F5 +**Account evidence:** @falconortiz Studio + API 2026-09-08 +**Code:** https://github.com/xai-org/x-algorithm @ `902a06f` +**Sender when approved:** GitHub **falconortiz** · SpaceXAI owns merge +**Pattern:** `docs/BIDIRECTIONAL_BOOST_CHANGE.md` (FS → A/B → ship) +**Status:** HOLD until Falcon go +**Eng:** Head of Engineering (IRE) — hardened packaging + +--- + +## 1. POV — is a mixed PR the right vehicle? + +**Yes — as one coherent cover / program with layered diffs, not one mega-commit.** + +| Layer | Role | Land now? | +|-------|------|-----------| +| **Study (PR-D)** | Serve mix original vs RT vs quote; re-serve≥2×; negatives if exposed | **Yes** — instrumentation | +| **Solutions (PR-A + PR-F1)** | FollowAuthor/ProfileClick A/B; prefer-original For You dedup | **Yes** — FS A/B on | +| **Structure (PR-F5)** | ControlledSourceReexposure — full FS + filter sketch | **Yes code / default off** — treatment arms only after D baseline | + +**Not in this ask as substitutes:** open F2b (burn off) or F4 (post-48h age prolong). F5 is the controlled alternative. +**Not pertinent:** study-only with no F5 sketch · F5-only without A/D/F1 · enabling F5 treatment before serve-mix baseline. + +**Projection:** Land D + A/F1 A/B + F5 params/filter **flag off** in one program. Keep F5 treatment off for ~1–2 experiment cycles until serve-mix baseline exists, then turn K=2/3 arms. Same “creator reach equilibrium” story — three layers, one cover. + +--- + +## 2. Problem statement (metrics-backed) + +1. **Conversion:** cohort follow/imp **0.00636%** vs Batch B **0.0644%** (~10×); FollowAuthor=4 vs CopyLink=20; ProfileClick=0. +2. **OCR:** 180/500 VF · 49.1K/500K HT; rolling floor ~**5,556** verified HT/day at η=3.78% or NEVER. +3. **Hard TL:** Age 48h · RT XOR original · source burn after serve · quotes bypass RT-dedup. +4. **Proxy:** high-amp strata follow/imp **0.0678%** vs low-amp megaviral **0.0053%**; serve % RT-vs-original **unmeasurable** today. +5. **Smoke:** current burn blocks niche re-see; F5 restores K=2–3 IN nudges without 20-RT flood (`e2e_smoke_f5/` 22/22 PASS sim). + +Hypothesis floors only — not global SLOs. + +--- + +## 3. F5 structure (implementable, default off) + +### 3.1 Placement +Policy **override on seen/served source burn** for a source_id when F5 allows re-serve — **not** an AgeFilter rewrite and **not** OON RT enable. + +Order of intent: +1. Same-slate ≤1 (existing RT dedup + **F1** prefer-original). +2. Seen/served burn as today when F5 off (K≡1). +3. When F5 on: re-admit source for a later request if K/gap/IN rules pass. Prefer **original** candidate if present; else IN amplifier card of that source. + +### 3.2 Feature Store params (proposed) + +| Param | Type | Default (ship) | Treatment arms | +|-------|------|----------------|----------------| +| `EnableControlledSourceReexposure` | bool | **false** | true | +| `ControlledSourceReexposureK` | u32 | **1** (≡ today’s no re-see) | **2**, **3** | +| `ControlledSourceReexposureGapRequests` | u32 | 1 | 1–3 | +| `ControlledSourceReexposureInNetworkOnly` | bool | **true** | keep true for first ship | +| `ControlledSourceReexposureIncludeQuotes` | bool | **true** | **fixed true** — no false arm (HoE) | + +### 3.3 `source_key` (quotes required) + +``` +if retweeted_tweet_id.is_some() -> retweeted_tweet_id +else if quoted_tweet_id.is_some() -> quoted_tweet_id # REQUIRED — smoke EXP− +else -> tweet_id +``` + +Quotes and RTs of the same original **share one cap bucket**. + +### 3.4 Acceptance +- Same request: ≤1 card per source (original / RT / quote). +- Across requests: ≤K serves per source / viewer / window. +- OON RT never used as F5 nudge. +- Attribution / logging: `get_original_tweet_id` + card_type ∈ {original, rt, quote}. +- No material ↑ report / mute / block vs control (needs PR-D). + +### 3.5 Files (sketch — SpaceXAI owns merge) +- Filter / policy near `previously_seen_*` / `previously_served_*` (re-admission) +- Params: `home-mixer/params/param.rs` +- Counters: Under the Hood / side effects (PR-D) +- Tests: mirror `e2e_smoke_f5/` EXP± + +Detail: `SPACEXAI_F5_IMPLEMENTATION.md` + +--- + +## 4. Mixed PR table of contents (external) +1. Cover one-pager (soft misalignment + hard rules + F5 north star) +2. **PR-D** study / telemetry +3. **PR-A** param sketch +4. **PR-F1** prefer-original (Following parity) +5. **PR-F5** implementation note — flag default **off** +6. Explicit non-asks: open F2b / F4 / E · no OCR invent · no merge claim · no VF SpamHighRecall loosen + +Companions already drafted: `SPACEXAI_COVER_TRANCHE1.md`, `SPACEXAI_PARAM_SKETCH_PR_A.md`, `SPACEXAI_PR_D_TELEMETRY_NOTE.md`, `SPACEXAI_RT_QUOTE_REQUEST.md` + +--- + +## 5. Risk / blast radius +| Risk | Mitigation | +|------|------------| +| Quote wall | Same source_key as RT; IncludeQuotes fixed true | +| Snowball | K + gap + ≤1/slate | +| Blind A/B | F5 treatment off until D serve-mix baseline | +| Safety VF | Do not loosen SpamHighRecall | +| Creator floors as SLOs | Label empirical only | + +## 6. Success projection (hypothesis, not SLO) +If η moves toward ~10% and follow/imp toward Batch B on non-viral mix, OCR HT path becomes more reachable; F5 targets **quality re-exposure**, not raw viral. Celebrity strata stay excluded from baseline reporting. diff --git a/docs/spacexai-ire-2026-09-08/SPACEXAI_PARAM_SKETCH_PR_A.md b/docs/spacexai-ire-2026-09-08/SPACEXAI_PARAM_SKETCH_PR_A.md new file mode 100644 index 00000000..0b1bd11e --- /dev/null +++ b/docs/spacexai-ire-2026-09-08/SPACEXAI_PARAM_SKETCH_PR_A.md @@ -0,0 +1,135 @@ +# Param sketch — PR-A FollowAuthor / ProfileClick rebalance + +**Mirrors:** `docs/BIDIRECTIONAL_BOOST_CHANGE.md` +**Repo:** https://github.com/xai-org/x-algorithm @ `902a06f` +**Primary file:** `home-mixer/params/param.rs` +**Scorer (unchanged):** `home-mixer/scorers/ranking_scorer.rs` apply terms ~418–447 +**No new hydrator.** FS keys already exist; weights already consumed. + +--- + +## Current defaults (`home-mixer/params/param.rs`) + +Verified @ `902a06f`: + +```rust +param!( + ProfileClickWeight, + f64, + "rust_home_mixer_profile_click_weight", + 0.0 +); +param!( + ShareViaCopyLinkWeight, + f64, + "rust_home_mixer_share_via_copy_link_weight", + 20.0 +); +param!( + FollowAuthorWeight, + f64, + "rust_home_mixer_follow_author_weight", + 4.0 +); +``` + +FS keys (already wired): + +- `rust_home_mixer_follow_author_weight` +- `rust_home_mixer_profile_click_weight` +- `rust_home_mixer_share_via_copy_link_weight` + +--- + +## Proposed A/B arms (Feature Store) + +| Arm | FollowAuthor | ProfileClick | CopyLink | Intent | +|-----|-------------:|-------------:|---------:|--------| +| **control** | 4 | 0 | 20 | ship today | +| **T1** | **12** | **1.0** | 20 | follow ~3×; profile on | +| **T2** | **20** | **2.0** | 20 | follow weight = CopyLink (equal-P) | +| **T3** | 20 | 2.0 | **10** | rebalance share vs follow | + +**Assignment (bidirectional July 2026 spirit):** small % on T1/T2/T3; majority control until early read. Prefer T1→T2 before enabling T3 (T3 also moves share volume). + +--- + +## Ship path (recommended) + +1. **FS A/B only** — do **not** commit default bumps in `param.rs` until a winning arm. +2. After win, document default change the way bidirectional did (reply boost → 15/20). + +### Illustrative post-win default bump (not the A/B vehicle) + +```diff +diff --git a/home-mixer/params/param.rs b/home-mixer/params/param.rs +--- a/home-mixer/params/param.rs ++++ b/home-mixer/params/param.rs +@@ + param!( + ProfileClickWeight, + f64, + "rust_home_mixer_profile_click_weight", +- 0.0 ++ 1.0 // replace with winning arm; FS override during A/B + ); +@@ + param!( + ShareViaCopyLinkWeight, + f64, + "rust_home_mixer_share_via_copy_link_weight", +- 20.0 ++ 20.0 // T3 only would set 10.0; keep 20 until T3 wins + ); +@@ + param!( + FollowAuthorWeight, + f64, + "rust_home_mixer_follow_author_weight", +- 4.0 ++ 12.0 // or 20.0 if T2/T3 wins + ); +``` + +--- + +## Scorer surface (unchanged) + +`ranking_scorer.rs` already includes: + +- `ProfileClickWeight * profile_click_score` (~426) +- `ShareViaCopyLinkWeight * share_via_copy_link_score` (~430–433) +- `FollowAuthorWeight * follow_author_score` (~447) + +PR-A adds **no** gates, filters, or hydrators. Follow remains one term in Σ, not a hard gate (that is PR-B, deferred). + +--- + +## Guardrails / metrics + +**Primary** + +- follow/imp (OON vs IN split if available) +- profile_click → follow (when funnel exists) + +**Safety** + +- report / mute / block rate +- copy-link / share volume (especially T3) + +**Hypothesis floors** (creator empirical — label as such, not global maxima) + +- follow/imp Batch B pocket **0.0644%** + +--- + +## Companion + +- Cover: `SPACEXAI_COVER_TRANCHE1.md` +- Telemetry: `SPACEXAI_PR_D_TELEMETRY_NOTE.md` + +## Out of scope this PR + +- OON low-P(follow) discount (PR-B) +- Author yield dampener (PR-C) +- Verified/OCR ranking term (PR-E) diff --git a/docs/spacexai-ire-2026-09-08/SPACEXAI_PR_D_TELEMETRY_NOTE.md b/docs/spacexai-ire-2026-09-08/SPACEXAI_PR_D_TELEMETRY_NOTE.md new file mode 100644 index 00000000..2ffef8d9 --- /dev/null +++ b/docs/spacexai-ire-2026-09-08/SPACEXAI_PR_D_TELEMETRY_NOTE.md @@ -0,0 +1,30 @@ +# PR-D — Creator-visible follow telemetry (Tranche 1 companion) + +**Goal:** one balance sheet for SpaceXAI + creators before heavier scorer PRs (B/C). +**Pair with:** PR-A param A/B (+ optional joint RT/quote send). Measure-first. +**Repo context:** x-algorithm @ `902a06f` — side effects already log follow_author; this is product/analytics surface, not a new Phoenix head. + +--- + +## Expose (aggregates only — not per-viewer P) + +1. **follow / impression** — 28d rolling, post + account +2. **Serve contribution mix** — % of ranked serves where top positive `w·P` term was CopyLink vs FollowAuthor (and optionally ProfileClick when non-zero) +3. **Verified HT / ALL** — only when OCR-eligible fields already exist in prod; **do not invent hydrators or fields** +4. **RT vs original serve mix** (PR-F2a) — for amplified posts: % serves as original `tweet_id` vs RT-of-original +5. **Quote attribution** (PR-F3) — follows on quoter vs profile clicks on nested author; QuotedClick contribution (weight default 0.05 today) + +## Why with PR-A / F1 + +Bidirectional-style A/B needs a shared KPI. Today Studio “New follows” has no verified split, and API organic has no follows-per-post. Without PR-D, SpaceXAI and creators cannot debug the same conversion / slot sheet the param and dedup changes are meant to move. + +## Non-goals + +- New Phoenix training label +- Publishing raw P(follow) to creators without product review +- Ranking / dedup changes (PR-A / PR-F1) +- Seen/served policy changes (PR-F2b — deferred) + +## Success + +Creators and SpaceXAI can answer, on the same numbers: “was this traffic share-dominated, follow-capable, or RT-slot-substituted?” before anyone ships OON gates or OCR terms. diff --git a/docs/spacexai-ire-2026-09-08/SPACEXAI_RT_QUOTE_REQUEST.md b/docs/spacexai-ire-2026-09-08/SPACEXAI_RT_QUOTE_REQUEST.md new file mode 100644 index 00000000..91898ca7 --- /dev/null +++ b/docs/spacexai-ire-2026-09-08/SPACEXAI_RT_QUOTE_REQUEST.md @@ -0,0 +1,52 @@ +# SpaceXAI request — RT/quote slot vs parallel original reach + +**Companion to:** Tranche 1 (PR-A FollowAuthor/ProfileClick rebalance + PR-D follow telemetry) +**Code:** https://github.com/xai-org/x-algorithm @ `902a06f` +**Pattern:** `docs/BIDIRECTIONAL_BOOST_CHANGE.md` (FS A/B → ship) +**Sender (when approved):** GitHub **falconortiz** · IRE · SpaceXAI owns merge +**Status:** HOLD until Falcon greenlights joint send with Tranche 1 + +--- + +## Plain language + +For You treats a **repost and its original as one content slot** (`RetweetDeduplicationFilter`: key = source tweet id, **first wins**). A celebrity RT can therefore **replace** the original in a viewer’s slate instead of stacking. + +Following already prefers the native tweet (`FollowingRetweetDeduplicationFilter`). For You does not. + +Quotes are separate posts and can run in parallel, but follow credit goes to the **quoter** (`follow_author` on quote author; `QuotedClickWeight` default **0.05**). + +There is **no** published knob that resets remaining reach to the last amplifier’s follower count. The “re-adjust after a smaller RT” feeling is explained by first-wins dedup + weaker IN distribution + optional seen/served related-id burn (`related_post_ids` = tweet + retweeted + reply-to; **not** quoted). + +## Ask (narrow) + +1. **PR-F1 (code):** Prefer original over RT in For You retweet dedup when both present — parity with Following. FS bool. +2. **Telemetry (fold into PR-D):** % of serves as original vs RT-of-original for amplified posts; quote follow vs nested-author profile click. +3. **Optional later (not this send):** seen/served policy so an RT impression does not permanently exclude the original without a fatigue model. + +## Success + +↑ original `tweet_id` serve share after celebrity RT; follow/imp on original not degraded; no duplicate-fatigue spike. + +## Evidence (hypothesis, one creator) + +@falconortiz `2076699535296856102` (Elon RT): ~1.1M imp → 24 follows (**0.0022%**); dominated cohort impressions with weak follow yield. Consistent with slot substitution + celebrity IN audience — **not** a counter reset, and **not** a global SLO. + +## Out of scope + +- Removing OON RT drop entirely +- OCR/verified ranking +- QuoteWeight / PR-B/C/E +- Claiming single-creator rates as global SLOs + +--- + +## Addendum — prolonged HT exposure vs amplifier volume + +**Gap:** Creators expect quote/RT volume to validate niche utility and **extend parallel** original Home exposure. Published formula does the opposite pressure: + +1. `AgeFilter` hard-caps eligibility at **48h** on `candidate.tweet_id` (`MAX_POST_AGE`). Quote/RT counts do not extend the original. New RT snowflakes can remain eligible after the original ages out. +2. Score uses `QuoteWeight·P(quote)` / `RetweetWeight·P(retweet)` — predicted actions — **not** `f(quote_count)` as a prolongation validator. +3. Served history records `source_tweet_id` with the RT (`ExcludeServedTweetIdsDuration` default 10m) → original can be excluded right after amplifier serve. + +**PR-F4 (deferred design note — not in this send):** amplification volume as positive evidence for **keeping** the original eligible/parallel; do not let post-48h distribution collapse solely onto amplifier cards if product wants prolonged original HT. Age/prolong policy is separate from **PR-F1** (same-slate prefer-original). Source burn on RT serve stays under **PR-F2b** (also deferred). diff --git a/docs/spacexai-ire-2026-09-08/STATUS.md b/docs/spacexai-ire-2026-09-08/STATUS.md new file mode 100644 index 00000000..c19b315f --- /dev/null +++ b/docs/spacexai-ire-2026-09-08/STATUS.md @@ -0,0 +1,64 @@ +# SpaceXAI pack — merge-ready STATUS + +**Base commit:** `902a06fd616ed815f660e5546d16d492fa1ca825` (`x-algorithm`) +**Date:** 2026-09-08 +**External send:** **HOLD** (per `READY_FOR_GO.md` — Falcon must say go) +**Do not push / open PRs from this box workstream.** + +## What landed (tree patches under `/workspace/x-algorithm`) + +| Pack | Status | Notes | +|------|--------|-------| +| **PR-A** | Comment-only | FS keys already present (Follow=4, Profile=0, CopyLink=20). Documented T1/T2/T3 arms near params; **defaults unchanged**. Pattern cites `docs/BIDIRECTIONAL_BOOST_CHANGE.md`. | +| **PR-F1** | Code + params + tests | `EnablePreferOriginalRetweetDedup` default **false**. Flag off = first-wins; flag on = Following prefer-original. | +| **PR-F5** | Code + params + tests | New `ControlledSourceReexposureFilter`; PreviouslyServed defers when Enable && K>1. Defaults: Enable=false, K=1, gap=0, InNetworkOnly=true. Quotes share `source_key` with RTs. | +| **PR-D** | Lightweight hook | `classify_serve_card` / `record_serve_card_type` (+ unit tests). Full Studio telemetry = product-side. | +| **Out** | — | F2b, F4, E, SpamHighRecall, AgeFilter, OON RT enable untouched. | + +## How to test + +### Standalone (green on any machine — REQUIRED) + +```bash +cd /workspace/x-algo-cache/merge-ready/spacexai-pack-tests +cargo test +``` + +**Result (this box):** **22 passed** (5 lib unit + 17 e2e integration), 0 failed. +Zero private deps. README explains monorepo gap. + +### Full home-mixer + +Needs SpaceXAI private monorepo (`xai_*` crates + Cargo workspace). Public OSS tree has no runnable home-mixer `Cargo.toml`. After merge into monorepo: + +```bash +# illustrative — monorepo paths may differ +cargo test -p home-mixer retweet_deduplication +cargo test -p home-mixer controlled_source_reexposure +cargo test -p home-mixer serve_card_classify +``` + +### Algorithm smoke (Python sim) + +```bash +python3 /workspace/x-algo-cache/e2e_smoke_f5/smoke_controlled_reexposure.py +# historically 22/22 PASS +``` + +## Patch export + +`/workspace/x-algo-cache/merge-ready/0001-spacexai-f1-f5-a-d.patch` +Apply from repo root: `git apply` / `git am` as appropriate. + +## Interaction note (F5 ↔ PreviouslyServed) + +- Enable=false **or** K≤1 → PreviouslyServed = today (drop on served related ids); F5 filter `enable()` false. +- Enable && K>1 → PreviouslyServed pass-through; F5 owns K / gap / IN-only / prefer-original / ≤1 per `source_key`. +- Wired in `phoenix_candidate_pipeline.rs` immediately after PreviouslyServedPostsFilter. +- AgeFilter / OONRetweetReplyFilter / VF unchanged. + +## Blockers + +- Cannot compile/run full home-mixer unit tests on this box (missing private crates) — mitigated by standalone crate. +- F5 request-gap uses `served_history` request indices when present; with gap default **0** the gate is a no-op until treatment arms set gap>0. +- Serve-count approximation from flat `served_ids` is sufficient for K gating; monorepo may refine with richer history later. diff --git a/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs b/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs index e8c9c397..9b6ccb37 100644 --- a/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs +++ b/home-mixer/candidate_pipeline/phoenix_candidate_pipeline.rs @@ -49,6 +49,7 @@ use crate::filters::oon_nsfw_simclusters_filter::OONNsfwSimclustersFilter; use crate::filters::oon_retweet_reply_filter::OONRetweetReplyFilter; use crate::filters::previously_seen_posts_backup_filter::PreviouslySeenPostsBackupFilter; use crate::filters::previously_seen_posts_filter::PreviouslySeenPostsFilter; +use crate::filters::controlled_source_reexposure_filter::ControlledSourceReexposureFilter; use crate::filters::previously_served_posts_filter::PreviouslyServedPostsFilter; use crate::filters::retweet_deduplication_filter::RetweetDeduplicationFilter; use crate::filters::self_tweet_filter::SelfTweetFilter; @@ -355,6 +356,8 @@ impl PhoenixCandidatePipeline { Box::new(PreviouslySeenPostsFilter), Box::new(PreviouslySeenPostsBackupFilter), Box::new(PreviouslyServedPostsFilter), + // PR-F5: armed only when EnableControlledSourceReexposure && K>1 + Box::new(ControlledSourceReexposureFilter), Box::new(ViewerMutedKeywordFilter::new()), Box::new(AuthorSocialgraphFilter), // Brazil 2026 election filter diff --git a/home-mixer/filters/controlled_source_reexposure_filter.rs b/home-mixer/filters/controlled_source_reexposure_filter.rs new file mode 100644 index 00000000..98d4918a --- /dev/null +++ b/home-mixer/filters/controlled_source_reexposure_filter.rs @@ -0,0 +1,280 @@ +//! PR-F5 ControlledSourceReexposure. +//! +//! Interaction with [`super::previously_served_posts_filter::PreviouslyServedPostsFilter`]: +//! - When `EnableControlledSourceReexposure` is false OR `MaxServesK <= 1`, +//! this filter is disabled and PreviouslyServed keeps today's burn behavior. +//! - When Enable && K > 1, PreviouslyServed pass-throughs served hits and this +//! filter owns: K cap, optional request gap, InNetworkOnly, prefer-original +//! within slate, ≤1 card per `source_key` (quotes share bucket with RTs). +//! AgeFilter / OON-RT / VF are untouched. + +use crate::models::candidate::PostCandidate; +use crate::models::query::ScoredPostsQuery; +use crate::params::{ + ControlledSourceReexposureInNetworkOnly, ControlledSourceReexposureMaxServesK, + ControlledSourceReexposureMinRequestGap, EnableControlledSourceReexposure, +}; +use crate::util::candidates_util::{related_post_ids_iter, source_key}; +use crate::util::serve_card_classify::record_serve_card_type; +use std::collections::{HashMap, HashSet}; +use xai_candidate_pipeline::filter::{Filter, FilterResult}; + +pub struct ControlledSourceReexposureFilter; + +impl Filter for ControlledSourceReexposureFilter { + fn enable(&self, query: &ScoredPostsQuery) -> bool { + query.params.get(EnableControlledSourceReexposure) + && query.params.get(ControlledSourceReexposureMaxServesK) > 1 + } + + fn filter( + &self, + query: &ScoredPostsQuery, + candidates: Vec, + ) -> FilterResult { + let k = query.params.get(ControlledSourceReexposureMaxServesK); + let min_gap = query.params.get(ControlledSourceReexposureMinRequestGap); + let in_network_only = query.params.get(ControlledSourceReexposureInNetworkOnly); + + let served_ids: HashSet = query.served_ids.iter().copied().collect(); + let mut serve_count: HashMap = HashMap::new(); + for &id in &query.served_ids { + *serve_count.entry(id).or_default() += 1; + } + + let history_len = query.served_history.len() as u32; + let mut last_serve_request: HashMap = HashMap::new(); + for (req_idx, sh) in query.served_history.iter().enumerate() { + for entry in &sh.entries { + for ids in entry.item_ids.iter().flatten() { + for opt in [ids.tweet_id, ids.source_tweet_id] { + if let Some(id) = opt { + last_serve_request.insert(id as u64, req_idx as u32); + } + } + } + } + } + + let mut groups: HashMap> = HashMap::new(); + let mut order: Vec = Vec::new(); + for c in candidates { + let sk = source_key(&c); + groups.entry(sk).or_insert_with(|| { + order.push(sk); + Vec::new() + }); + groups.get_mut(&sk).unwrap().push(c); + } + + let mut kept = Vec::new(); + let mut removed = Vec::new(); + + for sk in order { + let group = groups.remove(&sk).unwrap_or_default(); + let already_served = served_ids.contains(&sk) + || group + .iter() + .any(|c| related_post_ids_iter(c).any(|id| served_ids.contains(&id))); + + if already_served { + let count = *serve_count.get(&sk).unwrap_or(&0); + if count >= k { + removed.extend(group); + continue; + } + if min_gap > 0 { + if let Some(&last_req) = last_serve_request.get(&sk) { + if history_len.saturating_sub(last_req) < min_gap { + removed.extend(group); + continue; + } + } + } + + let (eligible, ineligible): (Vec<_>, Vec<_>) = + group.into_iter().partition(|c| { + if !in_network_only { + return true; + } + let is_amp = + c.retweeted_tweet_id.is_some() || c.quoted_tweet_id.is_some(); + if is_amp { + c.in_network == Some(true) + } else { + c.in_network != Some(false) + } + }); + removed.extend(ineligible); + if eligible.is_empty() { + continue; + } + let (chosen, leftovers) = pick_prefer_original(eligible); + removed.extend(leftovers); + if let Some(chosen) = chosen { + let _ = record_serve_card_type(&chosen); + kept.push(chosen); + } + } else { + let (chosen, leftovers) = pick_prefer_original(group); + removed.extend(leftovers); + if let Some(chosen) = chosen { + let _ = record_serve_card_type(&chosen); + kept.push(chosen); + } + } + } + + FilterResult { kept, removed } + } +} + +/// Prefer native/original over RT over quote; return ≤1 chosen + leftovers. +fn pick_prefer_original( + group: Vec, +) -> (Option, Vec) { + if group.is_empty() { + return (None, Vec::new()); + } + let natives: Vec<_> = group + .iter() + .filter(|c| c.retweeted_tweet_id.is_none() && c.quoted_tweet_id.is_none()) + .collect(); + let chosen_id = if let Some(n) = natives.first() { + n.tweet_id + } else { + let rts: Vec<_> = group + .iter() + .filter(|c| c.retweeted_tweet_id.is_some()) + .collect(); + rts.first() + .map(|c| c.tweet_id) + .unwrap_or(group[0].tweet_id) + }; + let mut chosen = None; + let mut leftovers = Vec::new(); + for c in group { + if c.tweet_id == chosen_id && chosen.is_none() { + chosen = Some(c); + } else { + leftovers.push(c); + } + } + (chosen, leftovers) +} + +#[cfg(test)] +mod tests { + use super::*; + use xai_feature_switches::{FeatureSwitches, Params, RecipientBuilder}; + + fn cand( + tweet_id: u64, + retweeted: Option, + quoted: Option, + in_network: Option, + ) -> PostCandidate { + PostCandidate { + tweet_id, + retweeted_tweet_id: retweeted, + quoted_tweet_id: quoted, + in_network, + ..Default::default() + } + } + + fn params(enable: bool, k: u32, gap: u32, in_only: bool) -> Params { + let mut results = FeatureSwitches::new(vec![]) + .unwrap() + .match_recipient(&RecipientBuilder::new().build()); + results.override_fs( + "rust_home_mixer_enable_controlled_source_reexposure".to_string(), + if enable { "true" } else { "false" }, + ); + results.override_fs( + "rust_home_mixer_controlled_source_reexposure_max_serves_k".to_string(), + &k.to_string(), + ); + results.override_fs( + "rust_home_mixer_controlled_source_reexposure_min_request_gap".to_string(), + &gap.to_string(), + ); + results.override_fs( + "rust_home_mixer_controlled_source_reexposure_in_network_only".to_string(), + if in_only { "true" } else { "false" }, + ); + results.into() + } + + fn query(served: Vec, enable: bool, k: u32, gap: u32, in_only: bool) -> ScoredPostsQuery { + ScoredPostsQuery { + served_ids: served, + params: params(enable, k, gap, in_only), + ..Default::default() + } + } + + #[test] + fn enable_false_when_flag_off_or_k_one() { + let f = ControlledSourceReexposureFilter; + assert!(!f.enable(&query(vec![], false, 2, 0, true))); + assert!(!f.enable(&query(vec![], true, 1, 0, true))); + assert!(f.enable(&query(vec![], true, 2, 0, true))); + } + + #[test] + fn enable_true_k2_allows_one_re_serve() { + let f = ControlledSourceReexposureFilter; + let q = query(vec![100], true, 2, 0, true); + let candidates = vec![ + cand(201, Some(100), None, Some(true)), + cand(202, Some(100), None, Some(true)), + ]; + let result = f.filter(&q, candidates); + assert_eq!(result.kept.len(), 1); + assert_eq!(result.kept[0].retweeted_tweet_id, Some(100)); + } + + #[test] + fn quotes_share_source_key_with_rt() { + let f = ControlledSourceReexposureFilter; + let q = query(vec![], true, 2, 0, true); + let candidates = vec![ + cand(201, Some(100), None, Some(true)), + cand(301, None, Some(100), Some(true)), + ]; + let result = f.filter(&q, candidates); + assert_eq!(result.kept.len(), 1, "≤1 per source across RT+quote"); + } + + #[test] + fn prefer_original_in_slate() { + let f = ControlledSourceReexposureFilter; + let q = query(vec![], true, 2, 0, true); + let candidates = vec![ + cand(201, Some(100), None, Some(true)), + cand(100, None, None, Some(true)), + ]; + let result = f.filter(&q, candidates); + assert_eq!(result.kept.len(), 1); + assert_eq!(result.kept[0].tweet_id, 100); + } + + #[test] + fn in_network_only_drops_oon_amp_on_reexposure() { + let f = ControlledSourceReexposureFilter; + let q = query(vec![100], true, 2, 0, true); + let candidates = vec![cand(201, Some(100), None, Some(false))]; + let result = f.filter(&q, candidates); + assert!(result.kept.is_empty()); + } + + #[test] + fn k_exhausted_drops_re_serve() { + let f = ControlledSourceReexposureFilter; + let q = query(vec![100, 100], true, 2, 0, true); + let candidates = vec![cand(201, Some(100), None, Some(true))]; + let result = f.filter(&q, candidates); + assert!(result.kept.is_empty()); + } +} diff --git a/home-mixer/filters/mod.rs b/home-mixer/filters/mod.rs index ce6231cc..340a7744 100644 --- a/home-mixer/filters/mod.rs +++ b/home-mixer/filters/mod.rs @@ -3,6 +3,7 @@ pub mod age_filter; pub mod ancillary_vf_filter; pub mod author_socialgraph_filter; pub mod brazil_2026_election_filter; +pub mod controlled_source_reexposure_filter; pub mod core_data_hydration_filter; pub mod dedup_conversation_filter; pub mod drop_duplicates_filter; diff --git a/home-mixer/filters/previously_served_posts_filter.rs b/home-mixer/filters/previously_served_posts_filter.rs index e3d421ec..d6091df2 100644 --- a/home-mixer/filters/previously_served_posts_filter.rs +++ b/home-mixer/filters/previously_served_posts_filter.rs @@ -1,6 +1,9 @@ use crate::models::candidate::PostCandidate; use crate::models::query::ScoredPostsQuery; -use crate::params::EnableServedFilterAllRequests; +use crate::params::{ + ControlledSourceReexposureMaxServesK, EnableControlledSourceReexposure, + EnableServedFilterAllRequests, +}; use crate::util::candidates_util::related_post_ids_iter; use std::collections::HashSet; use xai_candidate_pipeline::component_library::utils::client_utils::RequestContext::{ @@ -23,6 +26,18 @@ impl Filter for PreviouslyServedPostsFilter { query: &ScoredPostsQuery, candidates: Vec, ) -> FilterResult { + // PR-F5: when ControlledSourceReexposure is armed (Enable && K>1), defer + // served burn to ControlledSourceReexposureFilter (wired immediately after). + // Enable=false or K<=1 keeps today's drop-on-served behavior. + if query.params.get(EnableControlledSourceReexposure) + && query.params.get(ControlledSourceReexposureMaxServesK) > 1 + { + return FilterResult { + kept: candidates, + removed: Vec::new(), + }; + } + let served_ids: HashSet = query.served_ids.iter().copied().collect(); let (removed, kept): (Vec<_>, Vec<_>) = candidates diff --git a/home-mixer/filters/retweet_deduplication_filter.rs b/home-mixer/filters/retweet_deduplication_filter.rs index 5b747e9d..37c6c751 100644 --- a/home-mixer/filters/retweet_deduplication_filter.rs +++ b/home-mixer/filters/retweet_deduplication_filter.rs @@ -1,6 +1,8 @@ use crate::models::candidate::PostCandidate; use crate::models::query::ScoredPostsQuery; +use crate::params::EnablePreferOriginalRetweetDedup; use rustc_hash::FxHashSet; +use std::collections::HashSet; use xai_candidate_pipeline::filter::{Filter, FilterResult}; pub struct RetweetDeduplicationFilter; @@ -8,30 +10,77 @@ pub struct RetweetDeduplicationFilter; impl Filter for RetweetDeduplicationFilter { fn filter( &self, - _query: &ScoredPostsQuery, + query: &ScoredPostsQuery, candidates: Vec, ) -> FilterResult { - let mut seen_tweet_ids: FxHashSet = - FxHashSet::with_capacity_and_hasher(candidates.len(), Default::default()); - let mut kept = Vec::with_capacity(candidates.len()); - let mut removed = Vec::new(); - - for candidate in candidates { - let dedup_id = candidate.retweeted_tweet_id.unwrap_or(candidate.tweet_id); - if seen_tweet_ids.insert(dedup_id) { - kept.push(candidate); - } else { - removed.push(candidate); - } + if query.params.get(EnablePreferOriginalRetweetDedup) { + prefer_original_dedup(candidates) + } else { + first_wins_dedup(candidates) } + } +} + +/// Legacy For You behavior: first candidate for a source_id wins. +fn first_wins_dedup(candidates: Vec) -> FilterResult { + let mut seen_tweet_ids: FxHashSet = + FxHashSet::with_capacity_and_hasher(candidates.len(), Default::default()); + let mut kept = Vec::with_capacity(candidates.len()); + let mut removed = Vec::new(); + + for candidate in candidates { + let dedup_id = candidate.retweeted_tweet_id.unwrap_or(candidate.tweet_id); + if seen_tweet_ids.insert(dedup_id) { + kept.push(candidate); + } else { + removed.push(candidate); + } + } + + FilterResult { kept, removed } +} + +/// Following parity: prefer native/original over RT for the same source. +fn prefer_original_dedup(candidates: Vec) -> FilterResult { + let (retweets, native_tweets): (Vec<_>, Vec<_>) = candidates + .iter() + .cloned() + .partition(|c| c.retweeted_tweet_id.is_some()); + + let mut seen_tweet_ids: HashSet = HashSet::with_capacity(candidates.len()); + let mut kept_ids: HashSet = HashSet::with_capacity(candidates.len()); + for native in &native_tweets { + seen_tweet_ids.insert(native.tweet_id); + kept_ids.insert(native.tweet_id); + } + + for retweet in &retweets { + let source_id = retweet.retweeted_tweet_id.expect("partitioned as retweet"); + let ids = [retweet.tweet_id, source_id]; + if ids.iter().any(|id| seen_tweet_ids.contains(id)) { + continue; + } + seen_tweet_ids.extend(ids); + kept_ids.insert(retweet.tweet_id); + } - FilterResult { kept, removed } + let mut kept = Vec::with_capacity(kept_ids.len()); + let mut removed = Vec::new(); + for candidate in candidates { + if kept_ids.contains(&candidate.tweet_id) { + kept.push(candidate); + } else { + removed.push(candidate); + } } + + FilterResult { kept, removed } } #[cfg(test)] mod tests { use super::*; + use xai_feature_switches::{FeatureSwitches, Params, RecipientBuilder}; fn make_candidate(tweet_id: u64, retweeted_tweet_id: Option) -> PostCandidate { PostCandidate { @@ -41,6 +90,24 @@ mod tests { } } + fn params_prefer_original(enable: bool) -> Params { + let mut results = FeatureSwitches::new(vec![]) + .unwrap() + .match_recipient(&RecipientBuilder::new().build()); + results.override_fs( + "rust_home_mixer_enable_prefer_original_retweet_dedup".to_string(), + if enable { "true" } else { "false" }, + ); + results.into() + } + + fn query_with(enable_prefer_original: bool) -> ScoredPostsQuery { + ScoredPostsQuery { + params: params_prefer_original(enable_prefer_original), + ..Default::default() + } + } + #[tokio::test] async fn test_multiple_retweets_keeps_first() { let filter = RetweetDeduplicationFilter; @@ -150,4 +217,43 @@ mod tests { ); assert_eq!(result.removed.len(), 1); } + + #[tokio::test] + async fn test_prefer_original_flag_off_keeps_first_wins() { + let filter = RetweetDeduplicationFilter; + let query = query_with(false); + let candidates = vec![make_candidate(1, Some(100)), make_candidate(100, None)]; + let result = filter.filter(&query, candidates); + assert_eq!(result.kept.len(), 1); + assert_eq!(result.kept[0].tweet_id, 1); + } + + #[tokio::test] + async fn test_prefer_original_flag_on_original_wins_even_if_rt_first() { + let filter = RetweetDeduplicationFilter; + let query = query_with(true); + let candidates = vec![ + make_candidate(1, Some(100)), + make_candidate(100, None), + make_candidate(2, Some(100)), + ]; + let result = filter.filter(&query, candidates); + assert_eq!(result.kept.len(), 1); + assert_eq!(result.kept[0].tweet_id, 100); + assert!(result.kept[0].retweeted_tweet_id.is_none()); + assert_eq!(result.removed.len(), 2); + } + + #[tokio::test] + async fn test_prefer_original_flag_on_keeps_one_rt_when_no_native() { + let filter = RetweetDeduplicationFilter; + let query = query_with(true); + let candidates = vec![ + make_candidate(1, Some(100)), + make_candidate(2, Some(100)), + ]; + let result = filter.filter(&query, candidates); + assert_eq!(result.kept.len(), 1); + assert_eq!(result.kept[0].tweet_id, 1); + } } diff --git a/home-mixer/params/param.rs b/home-mixer/params/param.rs index 4df2ffa9..b021e137 100644 --- a/home-mixer/params/param.rs +++ b/home-mixer/params/param.rs @@ -340,6 +340,18 @@ param!( ); param!(ClickWeight, f64, "rust_home_mixer_click_weight", 0.4); param!(OpenLinkWeight, f64, "rust_home_mixer_open_link_weight", 0.2); +// --------------------------------------------------------------------------- +// PR-A (SpaceXAI pack): FollowAuthor / ProfileClick / CopyLink FS A/B +// Pattern: docs/BIDIRECTIONAL_BOOST_CHANGE.md — FS arms first; ship default +// only after a winning arm. Do NOT bump these defaults until then. +// +// Current ship defaults (control): FollowAuthor=4, ProfileClick=0, CopyLink=20. +// A/B arms (Feature Store only until win): +// T1: Follow=12, Profile=1, CopyLink=20 +// T2: Follow=20, Profile=2, CopyLink=20 +// T3: Follow=20, Profile=2, CopyLink=10 +// FS keys already exist; scorer already consumes them (ranking_scorer.rs). +// --------------------------------------------------------------------------- param!( ProfileClickWeight, f64, @@ -981,6 +993,42 @@ param!( true ); +// PR-F1: Prefer-original For You RetweetDedup (parity Following). Default false = first-wins. +param!( + EnablePreferOriginalRetweetDedup, + bool, + "rust_home_mixer_enable_prefer_original_retweet_dedup", + false +); + +// PR-F5: ControlledSourceReexposure. Enable=false and/or K=1 ≡ today's PreviouslyServed burn. +// Quotes always share source_key with RTs (IncludeQuotes fixed true — no false arm). +param!( + EnableControlledSourceReexposure, + bool, + "rust_home_mixer_enable_controlled_source_reexposure", + false +); +param!( + ControlledSourceReexposureMaxServesK, + u32, + "rust_home_mixer_controlled_source_reexposure_max_serves_k", + 1 +); +// Min requests between re-serves of the same source_key. 0 = no gap gate (landable default). +param!( + ControlledSourceReexposureMinRequestGap, + u32, + "rust_home_mixer_controlled_source_reexposure_min_request_gap", + 0 +); +param!( + ControlledSourceReexposureInNetworkOnly, + bool, + "rust_home_mixer_controlled_source_reexposure_in_network_only", + true +); + param!( EnableInventoryHoldout, bool, diff --git a/home-mixer/util/candidates_util.rs b/home-mixer/util/candidates_util.rs index c9d8711d..726dbc93 100644 --- a/home-mixer/util/candidates_util.rs +++ b/home-mixer/util/candidates_util.rs @@ -16,6 +16,18 @@ pub fn related_post_ids_iter(candidate: &PostCandidate) -> impl Iterator u64 { + if let Some(rt) = candidate.retweeted_tweet_id { + return rt; + } + if let Some(q) = candidate.quoted_tweet_id { + return q; + } + candidate.tweet_id +} + pub fn vqv_weight( query: &ScoredPostsQuery, candidate: &PostCandidate, @@ -107,4 +119,28 @@ mod tests { }; assert!((quoted_vqv_weight(&candidate, 10_000, 0.5, true)).abs() < 1e-9); } + + #[test] + fn source_key_prefers_retweeted_then_quoted_then_tweet() { + let rt = PostCandidate { + tweet_id: 1, + retweeted_tweet_id: Some(100), + quoted_tweet_id: Some(200), + ..Default::default() + }; + assert_eq!(source_key(&rt), 100); + + let quote = PostCandidate { + tweet_id: 2, + quoted_tweet_id: Some(100), + ..Default::default() + }; + assert_eq!(source_key("e), 100); + + let orig = PostCandidate { + tweet_id: 100, + ..Default::default() + }; + assert_eq!(source_key(&orig), 100); + } } diff --git a/home-mixer/util/mod.rs b/home-mixer/util/mod.rs index 75d041ec..232fec5d 100644 --- a/home-mixer/util/mod.rs +++ b/home-mixer/util/mod.rs @@ -13,3 +13,4 @@ pub mod tweet_type_metrics; pub mod url; pub mod urt; pub mod xds; +pub mod serve_card_classify; diff --git a/home-mixer/util/serve_card_classify.rs b/home-mixer/util/serve_card_classify.rs new file mode 100644 index 00000000..564c870c --- /dev/null +++ b/home-mixer/util/serve_card_classify.rs @@ -0,0 +1,79 @@ +use crate::models::candidate::PostCandidate; + +/// PR-D lightweight serve-mix classification. +/// Full Studio / creator telemetry is product-side; this is a pure hook for +/// counters / tracing only (no analytics surface required in home-mixer). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ServeCardType { + Original, + Rt, + Quote, +} + +/// Classify a serve card as original / rt / quote from candidate fields. +/// RT wins over quote if both were somehow set (RT is the outer card). +pub fn classify_serve_card(candidate: &PostCandidate) -> ServeCardType { + if candidate.retweeted_tweet_id.is_some() { + ServeCardType::Rt + } else if candidate.quoted_tweet_id.is_some() { + ServeCardType::Quote + } else { + ServeCardType::Original + } +} + +/// Side-effect stub: would increment serve-mix counters; logs at debug for now. +pub fn record_serve_card_type(candidate: &PostCandidate) -> ServeCardType { + let card = classify_serve_card(candidate); + tracing::debug!( + tweet_id = candidate.tweet_id, + card_type = ?card, + "spacexai.serve_card_type" + ); + card +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classifies_original_rt_quote() { + assert_eq!( + classify_serve_card(&PostCandidate { + tweet_id: 1, + ..Default::default() + }), + ServeCardType::Original + ); + assert_eq!( + classify_serve_card(&PostCandidate { + tweet_id: 2, + retweeted_tweet_id: Some(100), + ..Default::default() + }), + ServeCardType::Rt + ); + assert_eq!( + classify_serve_card(&PostCandidate { + tweet_id: 3, + quoted_tweet_id: Some(100), + ..Default::default() + }), + ServeCardType::Quote + ); + } + + #[test] + fn rt_wins_over_quote_if_both_set() { + assert_eq!( + classify_serve_card(&PostCandidate { + tweet_id: 4, + retweeted_tweet_id: Some(100), + quoted_tweet_id: Some(200), + ..Default::default() + }), + ServeCardType::Rt + ); + } +}