|
| 1 | +// Precompute the dataset growth timeline from git history at build time. |
| 2 | +// |
| 3 | +// The homepage History section used to call the GitHub API live and could only |
| 4 | +// show the last ~7 commits (per_page=8 + slice(7)), so older syncs were invisible |
| 5 | +// and the unauthenticated API rate limit (60/h) made it fragile. Instead we walk |
| 6 | +// the full git history of `site/public/v1/index.json` once during the deploy build |
| 7 | +// and emit `site/public/v1/history.json` — a single static file the page reads. |
| 8 | +// |
| 9 | +// Build-only artifact (gitignored): regenerated on every Pages deploy, so it is |
| 10 | +// always complete and never churns data PRs. Requires a full-depth checkout |
| 11 | +// (fetch-depth: 0) to see old commits; degrades to an empty timeline otherwise. |
| 12 | + |
| 13 | +import { execFileSync } from "node:child_process"; |
| 14 | +import { mkdirSync, writeFileSync } from "node:fs"; |
| 15 | +import { dirname, resolve } from "node:path"; |
| 16 | +import { fileURLToPath } from "node:url"; |
| 17 | + |
| 18 | +const SITE_DIR = resolve(dirname(fileURLToPath(import.meta.url)), ".."); |
| 19 | +const REPO_ROOT = resolve(SITE_DIR, ".."); |
| 20 | +const TRACKED = "site/public/v1/index.json"; |
| 21 | +const OUT = resolve(SITE_DIR, "public/v1/history.json"); |
| 22 | +const REPO_URL = "https://github.com/GetTechAPI/TechAPI"; |
| 23 | + |
| 24 | +// Categories we sum into the record total (matches the public dump manifest). |
| 25 | +const ORDER = ["smartphones", "tablets", "watches", "pdas", "socs", "gpus", "cpus", "brands"]; |
| 26 | +// Keep the chart legible if the history ever grows large: downsample to at most |
| 27 | +// MAX points, always preserving the first (baseline) and last (latest) commits. |
| 28 | +const MAX = 40; |
| 29 | + |
| 30 | +function git(args) { |
| 31 | + return execFileSync("git", ["-C", REPO_ROOT, ...args], { |
| 32 | + encoding: "utf8", |
| 33 | + maxBuffer: 64 * 1024 * 1024, |
| 34 | + }); |
| 35 | +} |
| 36 | + |
| 37 | +function countsOf(manifest) { |
| 38 | + const counts = {}; |
| 39 | + let total = 0; |
| 40 | + for (const key of ORDER) { |
| 41 | + const n = manifest?.collections?.[key]?.count; |
| 42 | + if (typeof n === "number") { |
| 43 | + counts[key] = n; |
| 44 | + total += n; |
| 45 | + } |
| 46 | + } |
| 47 | + return { counts, total }; |
| 48 | +} |
| 49 | + |
| 50 | +function downsample(points) { |
| 51 | + if (points.length <= MAX) return points; |
| 52 | + const step = (points.length - 1) / (MAX - 1); |
| 53 | + const picked = []; |
| 54 | + const seen = new Set(); |
| 55 | + for (let i = 0; i < MAX; i++) { |
| 56 | + const idx = Math.round(i * step); |
| 57 | + if (!seen.has(idx)) { |
| 58 | + seen.add(idx); |
| 59 | + picked.push(points[idx]); |
| 60 | + } |
| 61 | + } |
| 62 | + return picked; |
| 63 | +} |
| 64 | + |
| 65 | +function buildPoints() { |
| 66 | + // %H sha, %cI committer ISO date, %s subject — 0x1f-separated, one line/commit. |
| 67 | + const raw = git(["log", "--format=%H%x1f%cI%x1f%s", "--", TRACKED]).trim(); |
| 68 | + if (!raw) return []; |
| 69 | + const commits = raw.split("\n").map((line) => { |
| 70 | + const [sha, date, ...rest] = line.split("\x1f"); |
| 71 | + return { sha, date, title: rest.join("\x1f") }; |
| 72 | + }); |
| 73 | + // git log is newest-first; the timeline reads oldest-first. |
| 74 | + commits.reverse(); |
| 75 | + |
| 76 | + const points = []; |
| 77 | + for (const c of commits) { |
| 78 | + let manifest; |
| 79 | + try { |
| 80 | + manifest = JSON.parse(git(["show", `${c.sha}:${TRACKED}`])); |
| 81 | + } catch { |
| 82 | + continue; // file absent/unparseable at this commit — skip it |
| 83 | + } |
| 84 | + const { counts, total } = countsOf(manifest); |
| 85 | + if (!total) continue; |
| 86 | + points.push({ |
| 87 | + sha: c.sha.slice(0, 7), |
| 88 | + date: c.date, |
| 89 | + title: (c.title || "Dataset sync").trim(), |
| 90 | + url: `${REPO_URL}/commit/${c.sha}`, |
| 91 | + total, |
| 92 | + counts, |
| 93 | + }); |
| 94 | + } |
| 95 | + return downsample(points); |
| 96 | +} |
| 97 | + |
| 98 | +function main() { |
| 99 | + let points = []; |
| 100 | + try { |
| 101 | + points = buildPoints(); |
| 102 | + } catch (err) { |
| 103 | + console.warn(`[build-history] git history unavailable: ${err.message}`); |
| 104 | + } |
| 105 | + mkdirSync(dirname(OUT), { recursive: true }); |
| 106 | + writeFileSync( |
| 107 | + OUT, |
| 108 | + JSON.stringify({ generated_at: new Date().toISOString(), schema: 1, points }, null, 2), |
| 109 | + ); |
| 110 | + console.log(`[build-history] wrote ${points.length} point(s) -> ${OUT}`); |
| 111 | +} |
| 112 | + |
| 113 | +main(); |
0 commit comments