diff --git a/.gitignore b/.gitignore index 2d0ec30..6d7ddc2 100644 --- a/.gitignore +++ b/.gitignore @@ -39,5 +39,6 @@ yarn-error.log* **/*.tar.gz **/*.tgz **/*.log +**/*.sqlite package-lock.json **/*.bun \ No newline at end of file diff --git a/README.md b/README.md index 7d33cfa..9bb7ea4 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,12 @@ It also requires different dates for different sports. Football uses YYYY, while `GET /schedule/basketball-men/d1/2023/02` +### Team Schedule + +Returns games for one school in one sport/division/season. + +`GET /team-schedule/michigan/basketball-men/d1/2025` + ### Brackets Tournament bracket for a given sport, division, and year, including live scores. diff --git a/package.json b/package.json index a8db21a..9b5610e 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "test": "echo \"Error: no test specified\" && exit 1", "dev": "bun --watch src/index.ts", "start": "NODE_ENV=production bun src/index.ts", - "lint": "biome check --write src" + "lint": "biome check --write src", + "team-schedule:ingest": "bun src/team-schedule/cli.ts" }, "dependencies": { "@elysiajs/openapi": "^1.4.11", diff --git a/src/index.ts b/src/index.ts index 5c8cb7a..0bd824c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,6 +22,7 @@ import { import type { NewScoreboardParams } from "./scoreboard/types"; import * as v from 'valibot'; import { validDivisions, validGameIds, validScoreboardSports, validSports, validYears } from "./schema"; +import { getTeamScheduleGames, openTeamScheduleDb } from "./team-schedule/db"; // 30 minute cache for most routes const cache_30m = new ExpiryMap(30 * 60 * 1000); @@ -43,6 +44,7 @@ const validRoutes = new Map([ ["game", cache_45s], ["scoreboard", cache_45s], ["schedule-alt", cache_30m], + ["team-schedule", cache_30m], ["news", cache_30m], ["brackets", cache_45s] ]); @@ -445,6 +447,31 @@ export const app = new Elysia() year: validYears, }) }) + .get("/team-schedule/:schoolSlug/:sport/:division/:season", async ({ cache, cacheKey, params }) => { + const db = openTeamScheduleDb(); + try { + const season = Number(params.season); + const rows = getTeamScheduleGames( + db, + params.schoolSlug, + params.sport, + params.division, + season + ); + const data = JSON.stringify(rows); + cache.set(cacheKey, data); + return data; + } finally { + db.close(); + } + }, { + params: v.object({ + schoolSlug: v.pipe(v.string(), v.regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/)), + sport: validSports, + division: validDivisions, + season: validYears, + }) + }) // scoreboard route to fetch data from data.ncaa.com json endpoint .get("/scoreboard/:sport/*", async ({ cache, cacheKey, params, set, status }) => { const sportCodes = newCodesBySport[params.sport]; diff --git a/src/openapi.ts b/src/openapi.ts index 7446bdf..424011e 100644 --- a/src/openapi.ts +++ b/src/openapi.ts @@ -452,6 +452,44 @@ export const openapiSpec = openapi({ }, ], }, + "/team-schedule/{schoolSlug}/{sport}/{division}/{season}": { + get: { + responses: {}, + summary: "Team schedule", + description: + "Team schedule for a school/sport/division/season. Returns one row per game with home and away teams.", + parameters: [ + { + name: "schoolSlug", + in: "path", + schema: { type: "string" }, + required: true, + examples: makeExamples(["michigan", "duke", "ucla"]), + }, + { + name: "sport", + in: "path", + schema: { type: "string" }, + required: true, + examples: makeExamples(["basketball-men", "basketball-women"]), + }, + { + name: "division", + in: "path", + schema: { type: "string" }, + required: true, + examples: makeExamples(["d1", "d2", "d3", "fbs", "fcs"]), + }, + { + name: "season", + in: "path", + schema: { type: "string" }, + required: true, + examples: makeExamples(["2025", "2024"]), + }, + ] as OpenAPIV3.ParameterObject[], + }, + }, "/schools-index": { get: { responses: {}, diff --git a/src/team-schedule/cli.ts b/src/team-schedule/cli.ts new file mode 100644 index 0000000..60a4aba --- /dev/null +++ b/src/team-schedule/cli.ts @@ -0,0 +1,80 @@ +import { getSeasonYear } from "../codes"; +import { getDefaultDbPath } from "./db"; +import { ingestTeamSchedule } from "./ingest"; + +function getArg(flag: string) { + const prefix = `${flag}=`; + const pair = Bun.argv.find((arg) => arg.startsWith(prefix)); + if (pair) { + return pair.slice(prefix.length); + } + + const index = Bun.argv.findIndex((arg) => arg === flag); + if (index === -1) { + return undefined; + } + return Bun.argv[index + 1]; +} + +function hasFlag(flag: string) { + return Bun.argv.includes(flag); +} + +function printUsage() { + console.log("Usage: bun src/team-schedule/cli.ts [options]"); + console.log(""); + console.log("Options:"); + console.log(" --sport default: basketball-men"); + console.log(" --division default: d1"); + console.log(` --season-year default: ${getSeasonYear(new Date())}`); + console.log(` --db-path default: ${getDefaultDbPath()}`); + console.log(" --max-dates process only first N dates"); + console.log(" --delay-ms delay between upstream fetches (default: 350)"); + console.log(" --dry-run fetch and map only, do not write sqlite"); +} + +if (hasFlag("--help") || hasFlag("-h")) { + printUsage(); + process.exit(0); +} + +const sport = getArg("--sport") ?? "basketball-men"; +const division = getArg("--division") ?? "d1"; +const seasonYearRaw = getArg("--season-year"); +const seasonYear = seasonYearRaw ? parseInt(seasonYearRaw, 10) : getSeasonYear(new Date()); +const maxDatesRaw = getArg("--max-dates"); +const maxDates = maxDatesRaw ? parseInt(maxDatesRaw, 10) : undefined; +const delayMsRaw = getArg("--delay-ms"); +const delayMs = delayMsRaw ? parseInt(delayMsRaw, 10) : 350; +const dryRun = hasFlag("--dry-run"); +const dbPath = getArg("--db-path") ?? getDefaultDbPath(); + +if (Number.isNaN(seasonYear)) { + throw new Error("Invalid --season-year value"); +} +if (maxDatesRaw && Number.isNaN(maxDates)) { + throw new Error("Invalid --max-dates value"); +} +if (delayMsRaw && Number.isNaN(delayMs)) { + throw new Error("Invalid --delay-ms value"); +} + +console.log(`Starting ingest for ${sport}/${division}/${seasonYear}`); +console.log(`Mode: ${dryRun ? "dry-run" : "write"}`); +console.log(`DB path: ${dbPath}`); +if (typeof maxDates === "number") { + console.log(`Date limit: ${maxDates}`); +} + +const result = await ingestTeamSchedule({ + sport, + division, + seasonYear, + dbPath, + maxDates, + delayMs, + dryRun, +}); + +console.log("Done"); +console.log(JSON.stringify(result, null, 2)); diff --git a/src/team-schedule/db.ts b/src/team-schedule/db.ts new file mode 100644 index 0000000..45cc9cd --- /dev/null +++ b/src/team-schedule/db.ts @@ -0,0 +1,292 @@ +import { mkdirSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { Database } from "bun:sqlite"; +import type { GameRecord, TeamRecord, TeamScheduleGame } from "./types"; + +const schema = ` +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS sports ( + id INTEGER PRIMARY KEY, + slug TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS teams ( + id INTEGER PRIMARY KEY, + slug TEXT NOT NULL UNIQUE, + pretty_name TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS games ( + id INTEGER PRIMARY KEY, + game_id INTEGER NOT NULL, + sport_id INTEGER NOT NULL, + division TEXT NOT NULL, + season INTEGER NOT NULL, + start_unix INTEGER NOT NULL, + home_team_id INTEGER NOT NULL, + away_team_id INTEGER NOT NULL, + UNIQUE (sport_id, division, season, game_id), + FOREIGN KEY (sport_id) REFERENCES sports(id), + FOREIGN KEY (home_team_id) REFERENCES teams(id), + FOREIGN KEY (away_team_id) REFERENCES teams(id) +); + +CREATE INDEX IF NOT EXISTS idx_games_game_id +ON games (game_id); + +CREATE INDEX IF NOT EXISTS idx_games_home_lookup +ON games (sport_id, season, division, home_team_id, start_unix); + +CREATE INDEX IF NOT EXISTS idx_games_away_lookup +ON games (sport_id, season, division, away_team_id, start_unix); +`; + +interface IdRow { + id: number; +} + +interface TeamIdRow { + id: number; + slug: string; +} + +interface TeamScheduleRow { + game_id: number; + start_unix: number; + home: string; + home_slug: string; + away: string; + away_slug: string; +} + +function migrateGamesTable(db: Database) { + const columns = db + .query("PRAGMA table_info(games)") + .all() as Array<{ name: string; pk: number }>; + + if (columns.length === 0) { + return; + } + + const hasIdColumn = columns.some((column) => column.name === "id"); + const gameIdIsPrimaryKey = columns.some((column) => column.name === "game_id" && column.pk === 1); + + if (hasIdColumn && !gameIdIsPrimaryKey) { + return; + } + + db.exec("BEGIN TRANSACTION;"); + try { + db.exec("ALTER TABLE games RENAME TO games_old;"); + db.exec(` + CREATE TABLE games ( + id INTEGER PRIMARY KEY, + game_id INTEGER NOT NULL, + sport_id INTEGER NOT NULL, + division TEXT NOT NULL, + season INTEGER NOT NULL, + start_unix INTEGER NOT NULL, + home_team_id INTEGER NOT NULL, + away_team_id INTEGER NOT NULL, + UNIQUE (sport_id, division, season, game_id), + FOREIGN KEY (sport_id) REFERENCES sports(id), + FOREIGN KEY (home_team_id) REFERENCES teams(id), + FOREIGN KEY (away_team_id) REFERENCES teams(id) + ); + `); + db.exec(` + INSERT OR REPLACE INTO games (game_id, sport_id, division, season, start_unix, home_team_id, away_team_id) + SELECT game_id, sport_id, division, season, start_unix, home_team_id, away_team_id + FROM games_old; + `); + db.exec("DROP TABLE games_old;"); + db.exec("COMMIT;"); + } catch (error) { + db.exec("ROLLBACK;"); + throw error; + } +} + +export function getDefaultDbPath() { + return resolve(process.cwd(), "data", "team-schedules.sqlite"); +} + +export function openTeamScheduleDb(dbPath = getDefaultDbPath()) { + mkdirSync(dirname(dbPath), { recursive: true }); + const db = new Database(dbPath, { create: true, strict: true }); + db.exec(schema); + migrateGamesTable(db); + return db; +} + +export function ensureSport(db: Database, slug: string) { + db.query("INSERT INTO sports (slug) VALUES (?) ON CONFLICT (slug) DO NOTHING").run(slug); + const row = db.query("SELECT id FROM sports WHERE slug = ?").get(slug) as IdRow | null; + if (!row) { + throw new Error(`Could not resolve sport id for ${slug}`); + } + return row.id; +} + +export function upsertTeams(db: Database, teams: TeamRecord[]) { + if (teams.length === 0) { + return 0; + } + + const uniqueTeams = new Map(); + for (const team of teams) { + uniqueTeams.set(team.slug, team); + } + + const upsert = db.query( + `INSERT INTO teams (slug, pretty_name) VALUES (?, ?) + ON CONFLICT (slug) DO UPDATE SET pretty_name = excluded.pretty_name` + ); + + const tx = db.transaction((rows: TeamRecord[]) => { + for (const row of rows) { + upsert.run(row.slug, row.pretty_name); + } + }); + + const rows = [...uniqueTeams.values()]; + tx(rows); + return rows.length; +} + +function getTeamIds(db: Database, slugs: string[]) { + if (slugs.length === 0) { + return new Map(); + } + + const uniqueSlugs = [...new Set(slugs)]; + const placeholders = uniqueSlugs.map(() => "?").join(", "); + const rows = db + .query(`SELECT id, slug FROM teams WHERE slug IN (${placeholders})`) + .all(...uniqueSlugs) as TeamIdRow[]; + + const teamIds = new Map(); + for (const row of rows) { + teamIds.set(row.slug, row.id); + } + return teamIds; +} + +export function upsertGames(db: Database, sportId: number, games: GameRecord[]) { + if (games.length === 0) { + return 0; + } + + const teamIds = getTeamIds( + db, + games.flatMap((game) => [game.home_slug, game.away_slug]) + ); + + const upsert = db.query( + `INSERT INTO games ( + game_id, + sport_id, + division, + season, + start_unix, + home_team_id, + away_team_id + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (sport_id, division, season, game_id) DO UPDATE SET + start_unix = excluded.start_unix, + home_team_id = excluded.home_team_id, + away_team_id = excluded.away_team_id` + ); + + const tx = db.transaction((rows: GameRecord[]) => { + for (const row of rows) { + const homeTeamId = teamIds.get(row.home_slug); + const awayTeamId = teamIds.get(row.away_slug); + + if (!homeTeamId || !awayTeamId) { + throw new Error(`Missing team id for game ${row.game_id}`); + } + + upsert.run( + row.game_id, + sportId, + row.division, + row.season, + row.start_unix, + homeTeamId, + awayTeamId + ); + } + }); + + tx(games); + return games.length; +} + +export function getTeamScheduleGames( + db: Database, + schoolSlug: string, + sportSlug: string, + division: string, + season: number +): TeamScheduleGame[] { + const sportRow = db.query("SELECT id FROM sports WHERE slug = ?").get(sportSlug) as IdRow | null; + if (!sportRow) { + return []; + } + + const teamRow = db.query("SELECT id FROM teams WHERE slug = ?").get(schoolSlug) as IdRow | null; + if (!teamRow) { + return []; + } + + const rows = db + .query( + `SELECT + g.game_id, + g.start_unix, + ht.pretty_name AS home, + ht.slug AS home_slug, + at.pretty_name AS away, + at.slug AS away_slug + FROM games g + JOIN teams ht ON g.home_team_id = ht.id + JOIN teams at ON g.away_team_id = at.id + WHERE g.sport_id = ? + AND g.season = ? + AND g.division = ? + AND g.home_team_id = ? + + UNION ALL + + SELECT + g.game_id, + g.start_unix, + ht.pretty_name AS home, + ht.slug AS home_slug, + at.pretty_name AS away, + at.slug AS away_slug + FROM games g + JOIN teams ht ON g.home_team_id = ht.id + JOIN teams at ON g.away_team_id = at.id + WHERE g.sport_id = ? + AND g.season = ? + AND g.division = ? + AND g.away_team_id = ? + + ORDER BY start_unix` + ) + .all( + sportRow.id, + season, + division, + teamRow.id, + sportRow.id, + season, + division, + teamRow.id + ) as TeamScheduleRow[]; + + return rows; +} diff --git a/src/team-schedule/ingest.ts b/src/team-schedule/ingest.ts new file mode 100644 index 0000000..2f9ebce --- /dev/null +++ b/src/team-schedule/ingest.ts @@ -0,0 +1,259 @@ +import { getDivisionCode, getSeasonYear, newCodesBySport } from "../codes"; +import { fetchGqlScoreboard } from "../scoreboard/scoreboard"; +import type { Contest } from "../scoreboard/types"; +import { ensureSport, openTeamScheduleDb, upsertGames, upsertTeams } from "./db"; +import type { GameRecord, IngestOptions, IngestResult, TeamRecord } from "./types"; + +function toSlug(value: string) { + return value + .toLowerCase() + .trim() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); +} + +function toUnixSeconds(startTimeEpoch: string | number | undefined, startDate: string | undefined) { + const raw = Number(startTimeEpoch); + if (Number.isFinite(raw) && raw > 0) { + return raw > 1_000_000_000_000 ? Math.floor(raw / 1000) : Math.floor(raw); + } + + if (startDate) { + const fallback = Date.parse(`${startDate}T00:00:00Z`); + if (!Number.isNaN(fallback)) { + return Math.floor(fallback / 1000); + } + } + + return 0; +} + +function toYyyyMmDd(contestDate: string) { + const [month, day, year] = contestDate.split("/"); + if (!month || !day || !year) { + throw new Error(`Invalid contestDate format: ${contestDate}`); + } + return `${year}/${month.padStart(2, "0")}/${day.padStart(2, "0")}`; +} + +function mapContestGame( + contest: Contest, + sport: string, + division: string, + season: number +): { game: GameRecord; teams: TeamRecord[] } | null { + const teams = contest.teams ?? []; + const home = teams.find((team) => team.isHome); + const away = teams.find((team) => !team.isHome); + const gameId = Number(contest.contestId); + + if (!home || !away || !Number.isInteger(gameId)) { + return null; + } + + const homeSlug = home.seoname || toSlug(home.nameShort || ""); + const awaySlug = away.seoname || toSlug(away.nameShort || ""); + + if (!homeSlug || !awaySlug) { + return null; + } + + return { + game: { + game_id: gameId, + sport, + division, + season, + start_unix: toUnixSeconds(contest.startTimeEpoch, contest.startDate), + home_slug: homeSlug, + home: home.nameShort || homeSlug, + away_slug: awaySlug, + away: away.nameShort || awaySlug, + }, + teams: [ + { + slug: homeSlug, + pretty_name: home.nameShort || homeSlug, + }, + { + slug: awaySlug, + pretty_name: away.nameShort || awaySlug, + }, + ], + }; +} + +function delay(ms: number) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +interface ScheduleDateEntry { + contestDate?: string; +} + +interface ScheduleDatesResponse { + data?: { + schedules?: { + games?: ScheduleDateEntry[]; + }; + }; +} + +async function fetchSeasonDates(sport: string, division: string, seasonYear: number) { + const sportData = newCodesBySport[sport as keyof typeof newCodesBySport]; + if (!sportData) { + throw new Error(`Unsupported sport: ${sport}`); + } + + const divisionCode = getDivisionCode(sport, division); + if (typeof sportData.code !== "string" || sportData.code.length === 0) { + throw new Error(`Unsupported scoreboard sport code for ${sport}`); + } + if (typeof divisionCode !== "number") { + throw new Error(`Unsupported division code for ${sport}/${division}`); + } + const extensions = encodeURIComponent( + JSON.stringify({ + persistedQuery: { + version: 1, + sha256Hash: "a25ad021179ce1d97fb951a49954dc98da150089f9766e7e85890e439516ffbf", + }, + }) + ); + const variables = encodeURIComponent( + JSON.stringify({ + sportCode: sportData.code, + division: Number(divisionCode), + seasonYear, + }) + ); + + const url = `https://sdataprod.ncaa.com/?extensions=${extensions}&queryName=NCAA_schedules_today_web&variables=${variables}`; + const response = await fetch(url); + + if (!response.ok) { + throw new Error(`Failed to fetch schedule dates (${response.status})`); + } + + const json = (await response.json()) as ScheduleDatesResponse; + const rawDates: string[] = (json.data?.schedules?.games ?? []) + .map((entry) => entry.contestDate) + .filter((value: string | undefined): value is string => typeof value === "string"); + + const dates = rawDates + .filter((date) => /^\d{2}\/\d{2}\/\d{4}$/.test(date)) + .map(toYyyyMmDd) + .sort(); + + if (dates.length === 0) { + throw new Error(`No daily contest dates available for ${sport}/${division}/${seasonYear}`); + } + + return [...new Set(dates)]; +} + +async function fetchGamesForDate( + sport: string, + division: string, + seasonYear: number, + contestDate: string +) { + const sportData = newCodesBySport[sport as keyof typeof newCodesBySport]; + if (!sportData) { + throw new Error(`Unsupported sport: ${sport}`); + } + + const divisionCode = getDivisionCode(sport, division); + if (typeof sportData.code !== "string" || sportData.code.length === 0) { + throw new Error(`Unsupported scoreboard sport code for ${sport}`); + } + if (typeof divisionCode !== "number") { + throw new Error(`Unsupported division code for ${sport}/${division}`); + } + const isFootball = sportData.code === "MFB"; + + if (isFootball) { + throw new Error("Football ingest is not supported in this script yet"); + } + + const scoreboardDate = new Date(contestDate); + const inferredSeasonYear = Number.isNaN(scoreboardDate.getTime()) + ? seasonYear + : getSeasonYear(scoreboardDate); + + const payload = await fetchGqlScoreboard({ + sportCode: sportData.code, + division: Number(divisionCode), + seasonYear: inferredSeasonYear, + contestDate, + }); + + const contests: Contest[] = payload?.data?.contests ?? []; + const mapped = contests + .map((contest) => mapContestGame(contest, sport, division, seasonYear)) + .filter((value): value is { game: GameRecord; teams: TeamRecord[] } => value !== null); + + return { + contests, + games: mapped.map((entry) => entry.game), + teams: mapped.flatMap((entry) => entry.teams), + }; +} + +export async function ingestTeamSchedule(options: IngestOptions): Promise { + const { + sport, + division, + seasonYear, + dbPath, + dryRun = false, + delayMs = 350, + maxDates, + } = options; + + const allDates = await fetchSeasonDates(sport, division, seasonYear); + const dates = typeof maxDates === "number" && maxDates > 0 ? allDates.slice(0, maxDates) : allDates; + + const db = dryRun ? null : openTeamScheduleDb(dbPath); + let datesProcessed = 0; + let contestsProcessed = 0; + let gamesPrepared = 0; + let gamesWritten = 0; + let teamsUpserted = 0; + + try { + const sportId = db ? ensureSport(db, sport) : 0; + + for (const date of dates) { + const { contests, games, teams } = await fetchGamesForDate(sport, division, seasonYear, date); + contestsProcessed += contests.length; + gamesPrepared += games.length; + datesProcessed++; + + if (db) { + teamsUpserted += upsertTeams(db, teams); + gamesWritten += upsertGames(db, sportId, games); + } + + if (delayMs > 0) { + await delay(delayMs); + } + } + } finally { + db?.close(); + } + + return { + sport, + division, + seasonYear, + datesFound: allDates.length, + datesProcessed, + contestsProcessed, + gamesPrepared, + gamesWritten, + teamsUpserted, + }; +} diff --git a/src/team-schedule/types.ts b/src/team-schedule/types.ts new file mode 100644 index 0000000..290234e --- /dev/null +++ b/src/team-schedule/types.ts @@ -0,0 +1,47 @@ +export interface TeamRecord { + slug: string; + pretty_name: string; +} + +export interface GameRecord { + game_id: number; + sport: string; + division: string; + season: number; + start_unix: number; + home_slug: string; + home: string; + away_slug: string; + away: string; +} + +export interface IngestOptions { + sport: string; + division: string; + seasonYear: number; + dbPath?: string; + delayMs?: number; + maxDates?: number; + dryRun?: boolean; +} + +export interface IngestResult { + sport: string; + division: string; + seasonYear: number; + datesFound: number; + datesProcessed: number; + contestsProcessed: number; + gamesPrepared: number; + gamesWritten: number; + teamsUpserted: number; +} + +export interface TeamScheduleGame { + game_id: number; + start_unix: number; + home: string; + home_slug: string; + away: string; + away_slug: string; +}