|
| 1 | +// Aggregate scan over the approval logs written by src/permission/approval-log.ts |
| 2 | +// (~/.corbits/projects/**/approvals.jsonl) — the data CL-5666 needed to exist |
| 3 | +// before approval volume could be measured at all. |
| 4 | +// |
| 5 | +// Reports: total asks, split by mode (auto vs interactive) and outcome, a |
| 6 | +// per-rule breakdown, settle-duration and display-delay percentiles (the |
| 7 | +// display delay is the CL-5664 signal — a queued gate arming its timeout |
| 8 | +// before the operator could see it), and a mega-chain count (segments >= |
| 9 | +// MEGA_CHAIN_SEGMENT_THRESHOLD). |
| 10 | +// |
| 11 | +// Prints only aggregate counts and timings, never a tool subject or command |
| 12 | +// text — the log itself never records either, so there is nothing to leak |
| 13 | +// here even by accident. |
| 14 | +// |
| 15 | +// Run: bun run scripts/approval-forensics.ts |
| 16 | + |
| 17 | +import { readdirSync, lstatSync, readFileSync } from "node:fs"; |
| 18 | +import { join } from "node:path"; |
| 19 | +import { homedir } from "node:os"; |
| 20 | + |
| 21 | +import { APPROVAL_LOG_FILE, type ApprovalRecord } from "../src/permission/approval-log.js"; |
| 22 | +import { MEGA_CHAIN_SEGMENT_THRESHOLD } from "../src/permission/classify.js"; |
| 23 | + |
| 24 | +// lstat, and skip symlinks: session dirs carry a `latest` symlink to a real |
| 25 | +// session, and following it double-counts every record in that session. |
| 26 | +function findAll(dir: string, name: string, out: string[]): void { |
| 27 | + let entries: string[]; |
| 28 | + try { |
| 29 | + entries = readdirSync(dir); |
| 30 | + } catch { |
| 31 | + return; |
| 32 | + } |
| 33 | + for (const entry of entries) { |
| 34 | + const path = join(dir, entry); |
| 35 | + let info: ReturnType<typeof lstatSync>; |
| 36 | + try { |
| 37 | + info = lstatSync(path); |
| 38 | + } catch { |
| 39 | + continue; |
| 40 | + } |
| 41 | + if (info.isSymbolicLink()) continue; |
| 42 | + if (info.isDirectory()) findAll(path, name, out); |
| 43 | + else if (entry === name) out.push(path); |
| 44 | + } |
| 45 | +} |
| 46 | + |
| 47 | +function percentile(sorted: readonly number[], p: number): number { |
| 48 | + if (sorted.length === 0) return 0; |
| 49 | + const index = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length)); |
| 50 | + return sorted[index]!; |
| 51 | +} |
| 52 | + |
| 53 | +interface Bucket { |
| 54 | + count: number; |
| 55 | + byOutcome: Map<string, number>; |
| 56 | + byMode: Map<string, number>; |
| 57 | + durations: number[]; |
| 58 | + displayDelays: number[]; |
| 59 | + megaChains: number; |
| 60 | +} |
| 61 | + |
| 62 | +function emptyBucket(): Bucket { |
| 63 | + return { |
| 64 | + count: 0, |
| 65 | + byOutcome: new Map(), |
| 66 | + byMode: new Map(), |
| 67 | + durations: [], |
| 68 | + displayDelays: [], |
| 69 | + megaChains: 0, |
| 70 | + }; |
| 71 | +} |
| 72 | + |
| 73 | +const root = join(homedir(), ".corbits", "projects"); |
| 74 | +const files: string[] = []; |
| 75 | +findAll(root, APPROVAL_LOG_FILE, files); |
| 76 | + |
| 77 | +const buckets = new Map<string, Bucket>(); |
| 78 | +const sessionsByRule = new Map<string, Set<string>>(); |
| 79 | +let records = 0; |
| 80 | +let malformed = 0; |
| 81 | + |
| 82 | +for (const file of files) { |
| 83 | + let lines: string[]; |
| 84 | + try { |
| 85 | + lines = readFileSync(file, "utf8").split("\n"); |
| 86 | + } catch { |
| 87 | + continue; |
| 88 | + } |
| 89 | + for (const line of lines) { |
| 90 | + if (line.trim().length === 0) continue; |
| 91 | + let record: ApprovalRecord; |
| 92 | + try { |
| 93 | + record = JSON.parse(line) as ApprovalRecord; |
| 94 | + } catch { |
| 95 | + malformed++; |
| 96 | + continue; |
| 97 | + } |
| 98 | + if (typeof record.tool !== "string" || typeof record.outcome !== "string") { |
| 99 | + malformed++; |
| 100 | + continue; |
| 101 | + } |
| 102 | + records++; |
| 103 | + const key = record.tool; |
| 104 | + let bucket = buckets.get(key); |
| 105 | + if (bucket === undefined) { |
| 106 | + bucket = emptyBucket(); |
| 107 | + buckets.set(key, bucket); |
| 108 | + } |
| 109 | + bucket.count++; |
| 110 | + bucket.byOutcome.set(record.outcome, (bucket.byOutcome.get(record.outcome) ?? 0) + 1); |
| 111 | + bucket.byMode.set(record.mode, (bucket.byMode.get(record.mode) ?? 0) + 1); |
| 112 | + if (typeof record.durationMs === "number") bucket.durations.push(record.durationMs); |
| 113 | + if (typeof record.displayDelayMs === "number") bucket.displayDelays.push(record.displayDelayMs); |
| 114 | + if ((record.segments ?? 0) >= MEGA_CHAIN_SEGMENT_THRESHOLD) bucket.megaChains++; |
| 115 | + |
| 116 | + // Duplicate-rate proxy: how often the same rule fires more than once per |
| 117 | + // session file (a session repeatedly asking for something it was already |
| 118 | + // told no/yes to under a different subject). |
| 119 | + if (record.rule !== undefined) { |
| 120 | + const sessions = sessionsByRule.get(record.rule) ?? new Set<string>(); |
| 121 | + sessions.add(file); |
| 122 | + sessionsByRule.set(record.rule, sessions); |
| 123 | + } |
| 124 | + } |
| 125 | +} |
| 126 | + |
| 127 | +console.log(`approval logs: ${files.length}`); |
| 128 | +console.log(`records: ${records}${malformed > 0 ? ` (${malformed} malformed, skipped)` : ""}`); |
| 129 | +if (records === 0) { |
| 130 | + console.log("\nNo approvals logged yet. Run some sessions first."); |
| 131 | + process.exit(0); |
| 132 | +} |
| 133 | + |
| 134 | +const rows = [...buckets.entries()].sort((a, b) => b[1].count - a[1].count); |
| 135 | +console.log( |
| 136 | + "\ntool n auto/interactive duration p50/p90/max displayDelay p50/p90/max megaChains", |
| 137 | +); |
| 138 | +for (const [key, bucket] of rows) { |
| 139 | + const durations = [...bucket.durations].sort((a, b) => a - b); |
| 140 | + const delays = [...bucket.displayDelays].sort((a, b) => a - b); |
| 141 | + const durDist = |
| 142 | + durations.length === 0 |
| 143 | + ? "-" |
| 144 | + : `${percentile(durations, 50)}/${percentile(durations, 90)}/${durations[durations.length - 1]!}`; |
| 145 | + const delayDist = |
| 146 | + delays.length === 0 |
| 147 | + ? "-" |
| 148 | + : `${percentile(delays, 50)}/${percentile(delays, 90)}/${delays[delays.length - 1]!}`; |
| 149 | + const autoCount = bucket.byMode.get("auto") ?? 0; |
| 150 | + const interactiveCount = bucket.byMode.get("interactive") ?? 0; |
| 151 | + console.log( |
| 152 | + `${key.padEnd(26)} ${String(bucket.count).padStart(3)} ${String(autoCount).padStart(4)}/${String(interactiveCount).padEnd(11)} ${durDist.padEnd(24)} ${delayDist.padEnd(24)} ${bucket.megaChains}`, |
| 153 | + ); |
| 154 | +} |
| 155 | + |
| 156 | +console.log("\nby outcome"); |
| 157 | +for (const [key, bucket] of rows) { |
| 158 | + const outcomes = [...bucket.byOutcome.entries()] |
| 159 | + .sort((a, b) => b[1] - a[1]) |
| 160 | + .map(([outcome, count]) => `${outcome}=${count}`) |
| 161 | + .join(" "); |
| 162 | + console.log(`${key.padEnd(26)} ${outcomes}`); |
| 163 | +} |
| 164 | + |
| 165 | +console.log("\nrule -> sessions that hit it at least once (duplicate-rate proxy)"); |
| 166 | +for (const [rule, sessions] of [...sessionsByRule.entries()].sort( |
| 167 | + (a, b) => b[1].size - a[1].size, |
| 168 | +)) { |
| 169 | + console.log(`${rule.padEnd(26)} ${sessions.size}`); |
| 170 | +} |
0 commit comments