From c137b29b7bbcd7943ba27b90983adb8d2122a20e Mon Sep 17 00:00:00 2001 From: Wasim Amiri <7220175+wasimxyz@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:05:14 -0700 Subject: [PATCH 1/3] Add global search over runs, files, and instruments (#121) * Add global search over runs, files, and instruments Introduce a keyboard-accessible global search (header field + Cmd/Ctrl+K) that queries a new /api/v1/search endpoint and renders grouped, highlighted, live-updating results across runs, files, and instruments. - Backend globalSearch() with parallel per-type queries, match-reason tracking, relevance ranking, and literal wildcard escaping. - pg_trgm GIN indexes (run_id, filename, display_name) via migration 0029, with extension creation wired into the drizzle-kit push paths. - cmdk-based modal with scope tabs, debounced+aborted fetch, recent searches (localStorage), and empty/no-results states. - Integration tests covering auth, min-length, nested-file matches, pattern matches, scoping, and literal special characters. Co-authored-by: Cursor * Fix search modal width and align header toolbar controls. Co-authored-by: Cursor * Lazy-load the global search palette to trim initial bundle size GlobalSearch (cmdk + result rows + recent-searches) is mounted on every authenticated page via the root layout header, so keep it out of each page's initial JS with next/dynamic and warm the chunk on hover/focus so opening still feels instant. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- web/app/api/v1/search/route.ts | 42 + web/app/layout.tsx | 12 +- web/components/search/global-search.tsx | 339 +++ web/components/search/highlight.tsx | 60 + web/components/search/search-result-item.tsx | 179 ++ web/components/search/search-trigger.tsx | 93 + web/components/search/use-recent-searches.ts | 79 + web/drizzle/0029_add_search_trgm_indexes.sql | 8 + web/drizzle/meta/0029_snapshot.json | 2188 ++++++++++++++++++ web/drizzle/meta/_journal.json | 7 + web/lib/api/search.ts | 330 +++ web/lib/db/schema.ts | 83 +- web/lib/search-constants.ts | 7 + web/scripts/reset-database.ts | 5 + web/tests/integration/global-setup.ts | 11 + web/tests/integration/search.test.ts | 178 ++ 16 files changed, 3590 insertions(+), 31 deletions(-) create mode 100644 web/app/api/v1/search/route.ts create mode 100644 web/components/search/global-search.tsx create mode 100644 web/components/search/highlight.tsx create mode 100644 web/components/search/search-result-item.tsx create mode 100644 web/components/search/search-trigger.tsx create mode 100644 web/components/search/use-recent-searches.ts create mode 100644 web/drizzle/0029_add_search_trgm_indexes.sql create mode 100644 web/drizzle/meta/0029_snapshot.json create mode 100644 web/lib/api/search.ts create mode 100644 web/lib/search-constants.ts create mode 100644 web/tests/integration/search.test.ts diff --git a/web/app/api/v1/search/route.ts b/web/app/api/v1/search/route.ts new file mode 100644 index 00000000..7d9443b6 --- /dev/null +++ b/web/app/api/v1/search/route.ts @@ -0,0 +1,42 @@ +import type { NextRequest } from "next/server"; +import { authorize } from "@/lib/api/auth"; +import { globalSearch, type SearchScope } from "@/lib/api/search"; + +// --------------------------------------------------------------------------- +// GET /api/v1/search?q=…&scope=all|runs|files|instruments +// +// Cross-entity global search powering the ⌘K palette. Returns grouped, +// relevance-ordered matches over runs, files, and instruments. There is no +// row-level scoping in Data Hub, so any caller with `runs:read` sees the same +// set the rest of the app exposes. +// --------------------------------------------------------------------------- + +const VALID_SCOPES: ReadonlySet = new Set([ + "all", + "runs", + "files", + "instruments", +]); + +function parseScope(raw: string | null): SearchScope { + return raw && VALID_SCOPES.has(raw as SearchScope) + ? (raw as SearchScope) + : "all"; +} + +export async function GET(request: NextRequest) { + const authResult = await authorize(request, "runs:read"); + if (authResult instanceof Response) { + return authResult; + } + + const { searchParams } = request.nextUrl; + const query = searchParams.get("q") ?? ""; + const scope = parseScope(searchParams.get("scope")); + + // The builder itself returns an empty result below the minimum length, so + // the guard here just avoids the DB round-trip for 0–1 char queries. + const result = await globalSearch({ query, scope }); + + return Response.json(result); +} diff --git a/web/app/layout.tsx b/web/app/layout.tsx index 3a95d393..f1a183fa 100644 --- a/web/app/layout.tsx +++ b/web/app/layout.tsx @@ -10,6 +10,7 @@ import { NotificationBell } from "@/components/notifications/notification-bell"; import { NotificationsProvider } from "@/components/notifications/notifications-provider"; import { PreviewDeploymentBanner } from "@/components/preview-deployment-banner"; import { ArchiveDownloadProvider } from "@/components/runs/archive-download-provider"; +import { SearchTrigger } from "@/components/search/search-trigger"; import { ThemeProvider } from "@/components/theme-provider"; import { SIDEBAR_COOKIE_NAME, @@ -145,9 +146,14 @@ export default async function RootLayout({ }} /> -
- - +
+
+ +
+
+ + +
{children} diff --git a/web/components/search/global-search.tsx b/web/components/search/global-search.tsx new file mode 100644 index 00000000..dea9818e --- /dev/null +++ b/web/components/search/global-search.tsx @@ -0,0 +1,339 @@ +"use client"; + +import { Command as CommandPrimitive } from "cmdk"; +import { Clock, SearchIcon, SearchX } from "lucide-react"; +import { useRouter } from "next/navigation"; +import { useCallback, useEffect, useState } from "react"; +import { + SearchFileRow, + SearchInstrumentRow, + SearchRunRow, +} from "@/components/search/search-result-item"; +import { useRecentSearches } from "@/components/search/use-recent-searches"; +import { + CommandGroup, + CommandItem, + CommandList, +} from "@/components/ui/command"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogTitle, +} from "@/components/ui/dialog"; +import type { + GlobalSearchResult, + SearchFileResult, + SearchInstrumentResult, + SearchRunResult, + SearchScope, +} from "@/lib/api/search"; +import { MIN_QUERY_LENGTH } from "@/lib/search-constants"; +import { cn } from "@/lib/utils"; + +const DEBOUNCE_MS = 200; + +const SCOPE_TABS: { id: SearchScope; label: string }[] = [ + { id: "all", label: "All" }, + { id: "runs", label: "Runs" }, + { id: "files", label: "Files" }, + { id: "instruments", label: "Instruments" }, +]; + +const EMPTY_RESULT: GlobalSearchResult = { + runs: [], + files: [], + instruments: [], + counts: { runs: 0, files: 0, instruments: 0, total: 0 }, +}; + +function runHref(run: SearchRunResult): string { + return `/instruments/${run.instrumentId}/runs/${encodeURIComponent(run.runId)}`; +} + +// Files have no standalone page, so deep-link to the parent run and pre-fill +// the run-detail files search (which already filters + highlights that table). +function fileHref(file: SearchFileResult): string { + return `/instruments/${file.instrumentId}/runs/${encodeURIComponent( + file.runId + )}?files_search=${encodeURIComponent(file.filename)}`; +} + +function instrumentHref(instrument: SearchInstrumentResult): string { + return `/instruments/${instrument.id}`; +} + +function Kbd({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ); +} + +export function GlobalSearch({ + open, + onOpenChange, +}: { + open: boolean; + onOpenChange: (open: boolean) => void; +}) { + const router = useRouter(); + const { recent, add: addRecent } = useRecentSearches(); + const [query, setQuery] = useState(""); + const [scope, setScope] = useState("all"); + const [result, setResult] = useState(EMPTY_RESULT); + const [loading, setLoading] = useState(false); + + const trimmed = query.trim(); + const isSearchable = trimmed.length >= MIN_QUERY_LENGTH; + + // Reset transient state whenever the modal closes so the next open starts + // clean (empty query, "All" tab, recent-searches view). + useEffect(() => { + if (!open) { + setQuery(""); + setScope("all"); + setResult(EMPTY_RESULT); + setLoading(false); + } + }, [open]); + + // Debounced, race-safe fetch. An AbortController cancels the in-flight + // request when the query/scope changes so stale responses can't overwrite + // fresher ones. + useEffect(() => { + if (!(open && isSearchable)) { + setResult(EMPTY_RESULT); + setLoading(false); + return; + } + + const controller = new AbortController(); + setLoading(true); + const timer = setTimeout(async () => { + try { + const params = new URLSearchParams({ q: trimmed, scope }); + const res = await fetch(`/api/v1/search?${params}`, { + signal: controller.signal, + }); + if (!res.ok) { + throw new Error(`Search failed: ${res.status}`); + } + const data = (await res.json()) as GlobalSearchResult; + setResult(data); + } catch (err) { + if (!(err instanceof DOMException && err.name === "AbortError")) { + setResult(EMPTY_RESULT); + } + } finally { + // Only the latest (non-aborted) request clears the spinner. + if (!controller.signal.aborted) { + setLoading(false); + } + } + }, DEBOUNCE_MS); + + return () => { + controller.abort(); + clearTimeout(timer); + }; + }, [open, trimmed, isSearchable, scope]); + + const navigate = useCallback( + (href: string) => { + addRecent(trimmed); + onOpenChange(false); + router.push(href); + }, + [addRecent, trimmed, onOpenChange, router] + ); + + const showRecent = !isSearchable; + const hasResults = result.counts.total > 0; + + return ( + + + Search Data Hub + + Search across runs, files, and instruments. + + + +
+ + {/* Radix Dialog moves focus to the first focusable element (this + input) on open, so no explicit autoFocus is needed. */} + + esc +
+ +
+ {SCOPE_TABS.map((tab) => ( + + ))} +
+ + {isSearchable ? ( +
+ {loading + ? "Searching…" + : `${result.counts.total} ${ + result.counts.total === 1 ? "result" : "results" + } for "${trimmed}"`} +
+ ) : null} + + + {showRecent ? ( + setQuery(value)} + recent={recent} + /> + ) : null} + + {isSearchable && !loading && !hasResults ? ( + + ) : null} + + {isSearchable && hasResults ? ( + <> + {result.runs.length > 0 ? ( + + {result.runs.map((run) => ( + navigate(runHref(run))} + value={`run:${run.id}`} + > + + + ))} + + ) : null} + + {result.files.length > 0 ? ( + + {result.files.map((file) => ( + navigate(fileHref(file))} + value={`file:${file.id}`} + > + + + ))} + + ) : null} + + {result.instruments.length > 0 ? ( + + {result.instruments.map((instrument) => ( + navigate(instrumentHref(instrument))} + value={`instrument:${instrument.id}`} + > + + + ))} + + ) : null} + + ) : null} + + +
+
+ + ↑↓ navigate + + + ↵ open + +
+ + ⌘K to open from anywhere + +
+ + + + ); +} + +function RecentSearches({ + recent, + onSelect, +}: { + recent: string[]; + onSelect: (value: string) => void; +}) { + if (recent.length === 0) { + return ( +
+ +

+ Search runs, files, or instruments +

+
+ ); + } + + return ( + + {recent.map((value) => ( + onSelect(value)} + // Prefix keeps recent-search values from colliding with result ids. + value={`recent:${value}`} + > + + {value} + + ))} + + ); +} + +function NoResults({ query }: { query: string }) { + return ( +
+ +

No results for "{query}"

+
+ ); +} diff --git a/web/components/search/highlight.tsx b/web/components/search/highlight.tsx new file mode 100644 index 00000000..7a924fb2 --- /dev/null +++ b/web/components/search/highlight.tsx @@ -0,0 +1,60 @@ +import { Fragment } from "react"; + +// Escapes regex metacharacters so the query is matched literally — a filename +// query like `.*jpg` must highlight those characters, not act as a wildcard. +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Renders `text` with every case-insensitive occurrence of `query` wrapped in + * an accent-colored highlight. This is the primary signal for *why* a result + * matched, so it's applied to every field where the query can appear (title, + * secondary line, and match-reason line). + */ +export function Highlight({ + text, + query, + className, +}: { + text: string; + query: string; + className?: string; +}) { + const trimmed = query.trim(); + if (!trimmed) { + return <>{text}; + } + + const lowerQuery = trimmed.toLowerCase(); + const markClassName = + className ?? + "rounded-[3px] bg-primary/15 px-0.5 text-primary dark:bg-primary/25"; + + // Split into segments and drop the empty strings the regex emits between + // adjacent matches. A cumulative character offset gives every segment a + // stable, unique key without relying on the array index. + let offset = 0; + const segments = text + .split(new RegExp(`(${escapeRegExp(trimmed)})`, "gi")) + .map((value) => { + const start = offset; + offset += value.length; + return { value, start }; + }) + .filter((segment) => segment.value !== ""); + + return ( + <> + {segments.map((segment) => + segment.value.toLowerCase() === lowerQuery ? ( + + {segment.value} + + ) : ( + {segment.value} + ) + )} + + ); +} diff --git a/web/components/search/search-result-item.tsx b/web/components/search/search-result-item.tsx new file mode 100644 index 00000000..374e97b3 --- /dev/null +++ b/web/components/search/search-result-item.tsx @@ -0,0 +1,179 @@ +"use client"; + +import { + Activity, + Cpu, + File as FileIcon, + FileSpreadsheet, + FileText, + Image as ImageIcon, + type LucideIcon, +} from "lucide-react"; +import { Highlight } from "@/components/search/highlight"; +import { WatcherStatusBadge } from "@/components/watchers/watcher-status-badge"; +import type { + SearchFileResult, + SearchInstrumentResult, + SearchRunResult, +} from "@/lib/api/search"; +import { cn, formatBytes, formatRelativeTime } from "@/lib/utils"; + +const IMAGE_EXTENSIONS = new Set([ + "png", + "jpg", + "jpeg", + "gif", + "webp", + "tif", + "tiff", + "bmp", + "nd2", +]); +const SPREADSHEET_EXTENSIONS = new Set(["csv", "tsv", "xls", "xlsx"]); +const TEXT_EXTENSIONS = new Set([ + "txt", + "md", + "json", + "xml", + "log", + "yaml", + "yml", +]); + +// Maps a filename to a type-appropriate glyph, falling back to a generic file +// icon for unmapped extensions. +function iconForFilename(filename: string): LucideIcon { + const ext = filename.split(".").pop()?.toLowerCase() ?? ""; + if (IMAGE_EXTENSIONS.has(ext)) { + return ImageIcon; + } + if (SPREADSHEET_EXTENSIONS.has(ext)) { + return FileSpreadsheet; + } + if (TEXT_EXTENSIONS.has(ext)) { + return FileText; + } + return FileIcon; +} + +// Shared row scaffold: leading icon, a flexible text column, and a right stat. +function ResultRowShell({ + icon: Icon, + children, + stat, +}: { + icon: LucideIcon; + children: React.ReactNode; + stat: React.ReactNode; +}) { + return ( + <> + +
{children}
+
+ {stat} +
+ + ); +} + +function pluralRuns(count: number): string { + return `${count} total ${count === 1 ? "run" : "runs"}`; +} + +export function SearchRunRow({ + result, + query, +}: { + result: SearchRunResult; + query: string; +}) { + const when = result.acquiredAt ?? result.createdAt; + return ( + + + + + + ·{" "} + {formatRelativeTime(when)} + + {result.matchReason === "file" && result.matchedFilename ? ( + + Contains{" "} + + + + + ) : null} + + ); +} + +export function SearchFileRow({ + result, + query, +}: { + result: SearchFileResult; + query: string; +}) { + return ( + + + + + + {result.instrumentName} + + {result.runId} + + + ); +} + +export function SearchInstrumentRow({ + result, + query, +}: { + result: SearchInstrumentResult; + query: string; +}) { + return ( + + } + > + + + + + {result.matchReason === "pattern" && result.matchedPattern ? ( + <> + Matches pattern{" "} + + + {" "} + · {pluralRuns(result.runCount)} + + ) : ( + pluralRuns(result.runCount) + )} + + + ); +} diff --git a/web/components/search/search-trigger.tsx b/web/components/search/search-trigger.tsx new file mode 100644 index 00000000..0016d3b9 --- /dev/null +++ b/web/components/search/search-trigger.tsx @@ -0,0 +1,93 @@ +"use client"; + +import { SearchIcon } from "lucide-react"; +import dynamic from "next/dynamic"; +import { useEffect, useState } from "react"; +import { Button } from "@/components/ui/button"; + +// `GlobalSearch` pulls in cmdk plus the result-row/highlight components, none +// of which are needed until the palette is actually opened. This trigger is +// mounted on every authenticated page (root layout header), so keeping it out +// of the initial bundle matters more than it would for a one-off dialog. +const GlobalSearch = dynamic( + () => import("@/components/search/global-search").then((m) => m.GlobalSearch), + { ssr: false } +); + +// Warms the module cache so opening the palette (click or ⌘K) feels instant +// once the user has shown intent by hovering/focusing the trigger. +function preloadGlobalSearch() { + import("@/components/search/global-search"); +} + +// Returns true when the keydown originated from an editable field, so a +// global ⌘K/Ctrl+K doesn't hijack a shortcut a text field might own. +function isEditableTarget(target: EventTarget | null): boolean { + if (!(target instanceof HTMLElement)) { + return false; + } + const tag = target.tagName; + return ( + tag === "INPUT" || + tag === "TEXTAREA" || + tag === "SELECT" || + target.isContentEditable + ); +} + +export function SearchTrigger() { + const [open, setOpen] = useState(false); + // Platform detection runs post-mount so the SSR'd markup (no hint) matches + // the first client render, avoiding a hydration mismatch. + const [shortcutHint, setShortcutHint] = useState(""); + + useEffect(() => { + const isMac = /mac|iphone|ipad|ipod/i.test(navigator.userAgent); + setShortcutHint(isMac ? "⌘K" : "Ctrl K"); + }, []); + + useEffect(() => { + function onKeyDown(event: KeyboardEvent) { + if ( + event.key.toLowerCase() !== "k" || + !(event.metaKey || event.ctrlKey) + ) { + return; + } + // When the palette is already open its own input handles keys; don't + // treat that as an editable-field bail-out. + if (!open && isEditableTarget(event.target)) { + return; + } + event.preventDefault(); + setOpen((prev) => !prev); + } + + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [open]); + + return ( + <> + + + + ); +} diff --git a/web/components/search/use-recent-searches.ts b/web/components/search/use-recent-searches.ts new file mode 100644 index 00000000..4e20fb8c --- /dev/null +++ b/web/components/search/use-recent-searches.ts @@ -0,0 +1,79 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; + +const STORAGE_KEY = "data-hub:recent-searches"; +const MAX_RECENT = 8; + +function read(): string[] { + if (typeof window === "undefined") { + return []; + } + try { + const raw = window.localStorage.getItem(STORAGE_KEY); + if (!raw) { + return []; + } + const parsed = JSON.parse(raw); + return Array.isArray(parsed) + ? parsed.filter((v): v is string => typeof v === "string") + : []; + } catch { + // Corrupt or unavailable storage — treat as no history. + return []; + } +} + +/** + * Session-persisted recent search queries backed by `localStorage`. Kept + * client-only and per-browser by design (v1 has no server-side history). + * Returns the list plus `add`/`clear` mutators; the list is capped at + * MAX_RECENT with most-recent-first ordering and case-insensitive dedup. + */ +export function useRecentSearches() { + const [recent, setRecent] = useState([]); + + // Hydrate after mount to avoid a server/client mismatch on the SSR'd shell. + useEffect(() => { + setRecent(read()); + }, []); + + const persist = useCallback((next: string[]) => { + setRecent(next); + try { + window.localStorage.setItem(STORAGE_KEY, JSON.stringify(next)); + } catch { + // Storage full or blocked — the in-memory list still works this session. + } + }, []); + + const add = useCallback( + (query: string) => { + const trimmed = query.trim(); + if (!trimmed) { + return; + } + setRecent((current) => { + const deduped = current.filter( + (q) => q.toLowerCase() !== trimmed.toLowerCase() + ); + const next = [trimmed, ...deduped].slice(0, MAX_RECENT); + try { + window.localStorage.setItem(STORAGE_KEY, JSON.stringify(next)); + } catch { + // ignore — see persist() + } + return next; + }); + }, + // persist intentionally unused here; add() writes inline to avoid a stale + // closure over `recent`. + [] + ); + + const clear = useCallback(() => { + persist([]); + }, [persist]); + + return { recent, add, clear }; +} diff --git a/web/drizzle/0029_add_search_trgm_indexes.sql b/web/drizzle/0029_add_search_trgm_indexes.sql new file mode 100644 index 00000000..332aa512 --- /dev/null +++ b/web/drizzle/0029_add_search_trgm_indexes.sql @@ -0,0 +1,8 @@ +-- Trigram matching support for global search. `gin_trgm_ops` below is only +-- available once this extension exists. Drizzle does not manage extensions, so +-- it is created here (and, for the `drizzle-kit push` paths used by local +-- reseed and integration tests, in reset-database.ts / the test global-setup). +CREATE EXTENSION IF NOT EXISTS pg_trgm;--> statement-breakpoint +CREATE INDEX "idx_files_filename_trgm" ON "files" USING gin ("filename" gin_trgm_ops) WHERE "files"."deleted_at" is null;--> statement-breakpoint +CREATE INDEX "idx_instrument_runs_run_id_trgm" ON "instrument_runs" USING gin ("run_id" gin_trgm_ops);--> statement-breakpoint +CREATE INDEX "idx_instruments_display_name_trgm" ON "instruments" USING gin ("display_name" gin_trgm_ops); \ No newline at end of file diff --git a/web/drizzle/meta/0029_snapshot.json b/web/drizzle/meta/0029_snapshot.json new file mode 100644 index 00000000..69c739f8 --- /dev/null +++ b/web/drizzle/meta/0029_snapshot.json @@ -0,0 +1,2188 @@ +{ + "id": "90447fa8-1d75-4bfa-a7b4-ae83c4f14121", + "prevId": "617a8c5b-dff4-4a2c-8e1a-fea5ecbace3b", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "token_type": { + "name": "token_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "session_state": { + "name": "session_state", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_accounts_user_id": { + "name": "idx_accounts_user_id", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_userId_user_id_fk": { + "name": "account_userId_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_provider_providerAccountId_pk": { + "name": "account_provider_providerAccountId_pk", + "columns": ["provider", "providerAccountId"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.archive_jobs": { + "name": "archive_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "instrument_run_id": { + "name": "instrument_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "archive_bucket": { + "name": "archive_bucket", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "archive_key": { + "name": "archive_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "archive_job_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "uq_archive_jobs_inflight": { + "name": "uq_archive_jobs_inflight", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"archive_jobs\".\"status\" in ('pending', 'building')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_archive_jobs_run_fingerprint_status": { + "name": "idx_archive_jobs_run_fingerprint_status", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "archive_jobs_instrument_run_id_instrument_runs_id_fk": { + "name": "archive_jobs_instrument_run_id_instrument_runs_id_fk", + "tableFrom": "archive_jobs", + "tableTo": "instrument_runs", + "columnsFrom": ["instrument_run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "archive_jobs_created_by_user_id_fk": { + "name": "archive_jobs_created_by_user_id_fk", + "tableFrom": "archive_jobs", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.files": { + "name": "files", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "instrument_run_id": { + "name": "instrument_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "relative_path": { + "name": "relative_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "s3_bucket": { + "name": "s3_bucket", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "s3_key": { + "name": "s3_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "category": { + "name": "category", + "type": "file_category", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'raw'" + }, + "status": { + "name": "status", + "type": "file_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'detected'" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "detected_at": { + "name": "detected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "upload_requested_at": { + "name": "upload_requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "file_created_at": { + "name": "file_created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "uq_files_instrument_run_id_relative_path": { + "name": "uq_files_instrument_run_id_relative_path", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "relative_path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"files\".\"relative_path\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "uq_files_active_instrument_run_id_filename": { + "name": "uq_files_active_instrument_run_id_filename", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "filename", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"files\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "uq_files_s3_key": { + "name": "uq_files_s3_key", + "columns": [ + { + "expression": "s3_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"files\".\"s3_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_files_instrument_run_id": { + "name": "idx_files_instrument_run_id", + "columns": [ + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_files_status_instrument_run_id": { + "name": "idx_files_status_instrument_run_id", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "instrument_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_files_upload_queue": { + "name": "idx_files_upload_queue", + "columns": [ + { + "expression": "upload_requested_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"files\".\"upload_requested_at\" is not null and \"files\".\"uploaded_at\" is null and \"files\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_files_metadata_gin": { + "name": "idx_files_metadata_gin", + "columns": [ + { + "expression": "metadata", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "idx_files_filename_trgm": { + "name": "idx_files_filename_trgm", + "columns": [ + { + "expression": "\"filename\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"files\".\"deleted_at\" is null", + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "files_instrument_run_id_instrument_runs_id_fk": { + "name": "files_instrument_run_id_instrument_runs_id_fk", + "tableFrom": "files", + "tableTo": "instrument_runs", + "columnsFrom": ["instrument_run_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.instrument_notification_subscriptions": { + "name": "instrument_notification_subscriptions", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instrument_id": { + "name": "instrument_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_instrument_notification_subscriptions_user_id": { + "name": "idx_instrument_notification_subscriptions_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "instrument_notification_subscriptions_user_id_user_id_fk": { + "name": "instrument_notification_subscriptions_user_id_user_id_fk", + "tableFrom": "instrument_notification_subscriptions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "instrument_notification_subscriptions_instrument_id_instruments_id_fk": { + "name": "instrument_notification_subscriptions_instrument_id_instruments_id_fk", + "tableFrom": "instrument_notification_subscriptions", + "tableTo": "instruments", + "columnsFrom": ["instrument_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "instrument_notification_subscriptions_user_id_instrument_id_pk": { + "name": "instrument_notification_subscriptions_user_id_instrument_id_pk", + "columns": ["user_id", "instrument_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.instrument_runs": { + "name": "instrument_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "instrument_id": { + "name": "instrument_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "instrument_run_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'lambda'" + }, + "watcher_id": { + "name": "watcher_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_by": { + "name": "deleted_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_instrument_runs_instrument_id_created_at": { + "name": "idx_instrument_runs_instrument_id_created_at", + "columns": [ + { + "expression": "instrument_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_instrument_runs_active": { + "name": "idx_instrument_runs_active", + "columns": [ + { + "expression": "instrument_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"instrument_runs\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_instrument_runs_active_acquired_at": { + "name": "idx_instrument_runs_active_acquired_at", + "columns": [ + { + "expression": "instrument_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(\"acquired_at\", \"created_at\") desc", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"instrument_runs\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_instrument_runs_metadata_gin": { + "name": "idx_instrument_runs_metadata_gin", + "columns": [ + { + "expression": "metadata", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "idx_instrument_runs_run_id_trgm": { + "name": "idx_instrument_runs_run_id_trgm", + "columns": [ + { + "expression": "\"run_id\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "instrument_runs_instrument_id_instruments_id_fk": { + "name": "instrument_runs_instrument_id_instruments_id_fk", + "tableFrom": "instrument_runs", + "tableTo": "instruments", + "columnsFrom": ["instrument_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "instrument_runs_watcher_id_watchers_id_fk": { + "name": "instrument_runs_watcher_id_watchers_id_fk", + "tableFrom": "instrument_runs", + "tableTo": "watchers", + "columnsFrom": ["watcher_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "instrument_runs_deleted_by_user_id_fk": { + "name": "instrument_runs_deleted_by_user_id_fk", + "tableFrom": "instrument_runs", + "tableTo": "user", + "columnsFrom": ["deleted_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "uq_instrument_runs_instrument_id_run_id": { + "name": "uq_instrument_runs_instrument_id_run_id", + "nullsNotDistinct": false, + "columns": ["instrument_id", "run_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.instruments": { + "name": "instruments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "instrument_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "instrument_type": { + "name": "instrument_type", + "type": "instrument_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'generic'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_instruments_display_name_trgm": { + "name": "idx_instruments_display_name_trgm", + "columns": [ + { + "expression": "\"display_name\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_preferences": { + "name": "notification_preferences", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "runs_all_muted": { + "name": "runs_all_muted", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "comments_attributed_enabled": { + "name": "comments_attributed_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "comments_participated_enabled": { + "name": "comments_participated_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "slack_runs_enabled": { + "name": "slack_runs_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "slack_comments_attributed_enabled": { + "name": "slack_comments_attributed_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "slack_comments_participated_enabled": { + "name": "slack_comments_participated_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "notification_preferences_user_id_user_id_fk": { + "name": "notification_preferences_user_id_user_id_fk", + "tableFrom": "notification_preferences", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notifications": { + "name": "notifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "notification_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "comment_id": { + "name": "comment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "read_at": { + "name": "read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_notifications_user_id_created_at": { + "name": "idx_notifications_user_id_created_at", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_notifications_user_id_unread": { + "name": "idx_notifications_user_id_unread", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"notifications\".\"read_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "notifications_user_id_user_id_fk": { + "name": "notifications_user_id_user_id_fk", + "tableFrom": "notifications", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "notifications_run_id_instrument_runs_id_fk": { + "name": "notifications_run_id_instrument_runs_id_fk", + "tableFrom": "notifications", + "tableTo": "instrument_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "notifications_comment_id_run_comments_id_fk": { + "name": "notifications_comment_id_run_comments_id_fk", + "tableFrom": "notifications", + "tableTo": "run_comments", + "columnsFrom": ["comment_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "notifications_actor_user_id_user_id_fk": { + "name": "notifications_actor_user_id_user_id_fk", + "tableFrom": "notifications", + "tableTo": "user", + "columnsFrom": ["actor_user_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_access_tokens": { + "name": "personal_access_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_prefix": { + "name": "token_prefix", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "ARRAY['*']::text[]" + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_personal_access_tokens_user_id": { + "name": "idx_personal_access_tokens_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_access_tokens_user_id_user_id_fk": { + "name": "personal_access_tokens_user_id_user_id_fk", + "tableFrom": "personal_access_tokens", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personal_access_tokens_token_hash_unique": { + "name": "personal_access_tokens_token_hash_unique", + "nullsNotDistinct": false, + "columns": ["token_hash"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.run_attributions": { + "name": "run_attributions", + "schema": "", + "columns": { + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_run_attributions_run_id": { + "name": "idx_run_attributions_run_id", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_run_attributions_user_id": { + "name": "idx_run_attributions_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "run_attributions_run_id_instrument_runs_id_fk": { + "name": "run_attributions_run_id_instrument_runs_id_fk", + "tableFrom": "run_attributions", + "tableTo": "instrument_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "run_attributions_user_id_user_id_fk": { + "name": "run_attributions_user_id_user_id_fk", + "tableFrom": "run_attributions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "run_attributions_run_id_user_id_pk": { + "name": "run_attributions_run_id_user_id_pk", + "columns": ["run_id", "user_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.run_comments": { + "name": "run_comments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "edited_at": { + "name": "edited_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_run_comments_run_id_created_at": { + "name": "idx_run_comments_run_id_created_at", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_run_comments_user_id": { + "name": "idx_run_comments_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "run_comments_run_id_instrument_runs_id_fk": { + "name": "run_comments_run_id_instrument_runs_id_fk", + "tableFrom": "run_comments", + "tableTo": "instrument_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "run_comments_user_id_user_id_fk": { + "name": "run_comments_user_id_user_id_fk", + "tableFrom": "run_comments", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "sessionToken": { + "name": "sessionToken", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires": { + "name": "expires", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_sessions_user_id": { + "name": "idx_sessions_user_id", + "columns": [ + { + "expression": "userId", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_userId_user_id_fk": { + "name": "session_userId_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.slack_channel_config": { + "name": "slack_channel_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "boolean", + "primaryKey": true, + "notNull": true, + "default": true + }, + "webhook_url": { + "name": "webhook_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "slack_channel_config_updated_by_user_id_fk": { + "name": "slack_channel_config_updated_by_user_id_fk", + "tableFrom": "slack_channel_config", + "tableTo": "user", + "columnsFrom": ["updated_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "slack_channel_config_singleton": { + "name": "slack_channel_config_singleton", + "value": "\"slack_channel_config\".\"id\" = true" + } + }, + "isRLSEnabled": false + }, + "public.slack_connections": { + "name": "slack_connections", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "slack_user_id": { + "name": "slack_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slack_team_id": { + "name": "slack_team_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slack_team_name": { + "name": "slack_team_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "connected_at": { + "name": "connected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "slack_connections_user_id_user_id_fk": { + "name": "slack_connections_user_id_user_id_fk", + "tableFrom": "slack_connections", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_admin": { + "name": "is_admin", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.watcher_events": { + "name": "watcher_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "watcher_id": { + "name": "watcher_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "watcher_event_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "timestamp": { + "name": "timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_watcher_events_watcher_id_timestamp": { + "name": "idx_watcher_events_watcher_id_timestamp", + "columns": [ + { + "expression": "watcher_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "timestamp", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_watcher_events_watcher_id_event_type": { + "name": "idx_watcher_events_watcher_id_event_type", + "columns": [ + { + "expression": "watcher_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "watcher_events_watcher_id_watchers_id_fk": { + "name": "watcher_events_watcher_id_watchers_id_fk", + "tableFrom": "watcher_events", + "tableTo": "watchers", + "columnsFrom": ["watcher_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.watcher_heartbeats": { + "name": "watcher_heartbeats", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "watcher_id": { + "name": "watcher_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "timestamp": { + "name": "timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "upload_mode": { + "name": "upload_mode", + "type": "upload_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "files_uploaded_since_last": { + "name": "files_uploaded_since_last", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "runs_reported_since_last": { + "name": "runs_reported_since_last", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "errors_since_last": { + "name": "errors_since_last", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "uptime_seconds": { + "name": "uptime_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_watcher_heartbeats_watcher_id_timestamp": { + "name": "idx_watcher_heartbeats_watcher_id_timestamp", + "columns": [ + { + "expression": "watcher_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "timestamp", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "watcher_heartbeats_watcher_id_watchers_id_fk": { + "name": "watcher_heartbeats_watcher_id_watchers_id_fk", + "tableFrom": "watcher_heartbeats", + "tableTo": "watchers", + "columnsFrom": ["watcher_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.watcher_release_config": { + "name": "watcher_release_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "boolean", + "primaryKey": true, + "notNull": true, + "default": true + }, + "latest_version": { + "name": "latest_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "min_supported_version": { + "name": "min_supported_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'stable'" + }, + "mandatory": { + "name": "mandatory", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "watcher_release_config_updated_by_user_id_fk": { + "name": "watcher_release_config_updated_by_user_id_fk", + "tableFrom": "watcher_release_config", + "tableTo": "user", + "columnsFrom": ["updated_by"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "watcher_release_config_singleton": { + "name": "watcher_release_config_singleton", + "value": "\"watcher_release_config\".\"id\" = true" + } + }, + "isRLSEnabled": false + }, + "public.watchers": { + "name": "watchers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "instrument_id": { + "name": "instrument_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "os_info": { + "name": "os_info", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "watcher_version": { + "name": "watcher_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config_checksum": { + "name": "config_checksum", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config_yaml": { + "name": "config_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_heartbeat_at": { + "name": "last_heartbeat_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "watcher_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'registered'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "uq_watchers_active_instrument_id": { + "name": "uq_watchers_active_instrument_id", + "columns": [ + { + "expression": "instrument_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"watchers\".\"deleted_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "watchers_instrument_id_instruments_id_fk": { + "name": "watchers_instrument_id_instruments_id_fk", + "tableFrom": "watchers", + "tableTo": "instruments", + "columnsFrom": ["instrument_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.archive_job_status": { + "name": "archive_job_status", + "schema": "public", + "values": ["pending", "building", "ready", "failed"] + }, + "public.file_category": { + "name": "file_category", + "schema": "public", + "values": ["raw", "processed"] + }, + "public.file_status": { + "name": "file_status", + "schema": "public", + "values": [ + "detected", + "upload_requested", + "uploaded", + "processing", + "completed", + "failed" + ] + }, + "public.instrument_run_source": { + "name": "instrument_run_source", + "schema": "public", + "values": ["lambda", "watcher"] + }, + "public.instrument_status": { + "name": "instrument_status", + "schema": "public", + "values": ["pending", "active", "inactive"] + }, + "public.instrument_type": { + "name": "instrument_type", + "schema": "public", + "values": [ + "generic", + "plate_reader", + "gel_doc", + "qpcr", + "tape_station", + "hina_microscope", + "epson_v700_scanner", + "instant_raman" + ] + }, + "public.notification_type": { + "name": "notification_type", + "schema": "public", + "values": ["run_created", "comment_attributed", "comment_participated"] + }, + "public.upload_mode": { + "name": "upload_mode", + "schema": "public", + "values": ["auto", "manual"] + }, + "public.watcher_event_type": { + "name": "watcher_event_type", + "schema": "public", + "values": [ + "watcher_started", + "watcher_stopped", + "file_uploaded", + "upload_failed", + "run_reported", + "config_synced", + "error", + "update_started", + "update_succeeded", + "update_failed" + ] + }, + "public.watcher_status": { + "name": "watcher_status", + "schema": "public", + "values": ["registered", "watching", "stopped"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/web/drizzle/meta/_journal.json b/web/drizzle/meta/_journal.json index d24a45ff..e683be17 100644 --- a/web/drizzle/meta/_journal.json +++ b/web/drizzle/meta/_journal.json @@ -204,6 +204,13 @@ "when": 1782935691822, "tag": "0028_add_slack_channel_config", "breakpoints": true + }, + { + "idx": 29, + "version": "7", + "when": 1783365397666, + "tag": "0029_add_search_trgm_indexes", + "breakpoints": true } ] } diff --git a/web/lib/api/search.ts b/web/lib/api/search.ts new file mode 100644 index 00000000..ee38b874 --- /dev/null +++ b/web/lib/api/search.ts @@ -0,0 +1,330 @@ +import { and, desc, eq, ilike, isNull, or, type SQL, sql } from "drizzle-orm"; +import { + getWatcherOnlineStatus, + type WatcherOnlineStatus, +} from "@/components/watchers/watcher-online-status"; +import { getInstrumentListWithCounts } from "@/lib/api/instruments"; +import { db } from "@/lib/db"; +import { + files, + instrumentRuns, + instruments, + runAttributions, + users, +} from "@/lib/db/schema"; +import { MIN_QUERY_LENGTH } from "@/lib/search-constants"; + +// Global search across the three top-level entities. Backed by the pg_trgm +// GIN indexes added in migration 0029 so the `ilike '%…%'` scans stay fast as +// the run/file tables grow (one instrument alone already carries 600+ runs). + +export type SearchScope = "all" | "runs" | "files" | "instruments"; + +// Per-group cap when searching everything at once ("All" tab). Matches the +// mockups. A scoped search (single tab) uses SCOPED_LIMIT so "Show all" +// surfaces a longer list without a dedicated results page. +const ALL_TAB_PER_GROUP = 5; +const SCOPED_LIMIT = 25; + +// Why a run surfaced. `run_id` is the run's own title (highest relevance); +// `file` means a contained filename matched (drives the "Contains …" line); +// `instrument`/`ran_by` are secondary metadata matches. +export type RunMatchReason = "run_id" | "file" | "instrument" | "ran_by"; + +export interface SearchRunResult { + acquiredAt: string | null; + createdAt: string; + fileCount: number; + id: string; + instrumentId: string; + instrumentName: string; + // Set only when `matchReason` is "file" — an example matching filename. + matchedFilename: string | null; + matchReason: RunMatchReason; + runId: string; + totalSizeBytes: number; + type: "run"; +} + +export interface SearchFileResult { + filename: string; + id: number; + instrumentId: string; + instrumentName: string; + runId: string; + sizeBytes: number | null; + type: "file"; +} + +export type InstrumentMatchReason = "name" | "pattern"; + +export interface SearchInstrumentResult { + displayName: string; + id: string; + lastWatcherHeartbeatAt: string | null; + // Set only when `matchReason` is "pattern" — the configured pattern that + // matched (e.g. "*.nd2"), so the client can highlight it. + matchedPattern: string | null; + matchReason: InstrumentMatchReason; + runCount: number; + status: "pending" | "active" | "inactive"; + type: "instrument"; + watcherStatus: WatcherOnlineStatus; +} + +export interface GlobalSearchResult { + counts: { + runs: number; + files: number; + instruments: number; + // Sum of the *visible* (capped) results — this is the "N results for …" + // figure the header shows, matching what the user actually sees. + total: number; + }; + files: SearchFileResult[]; + instruments: SearchInstrumentResult[]; + runs: SearchRunResult[]; +} + +// Escapes LIKE wildcards so user input matches literally. A filename like +// `.*jpg` or a pattern query must be treated as text, never as a LIKE/regex +// pattern. Mirrors the escaping already used in `buildRunListQuery`. +function escapeLike(query: string): string { + return query.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_"); +} + +const acquiredOrCreated = sql`coalesce(${instrumentRuns.acquiredAt}, ${instrumentRuns.createdAt})`; + +async function searchRuns( + query: string, + limit: number +): Promise { + const escaped = escapeLike(query); + const substring = `%${escaped}%`; + const prefix = `${escaped}%`; + + // A run matches on its own id, its instrument's name, a contained (active) + // filename, or an attributed user's name/email. + const fileMatch = sql`exists (select 1 from ${files} f where f.instrument_run_id = ${instrumentRuns.id} and f.deleted_at is null and f.filename ilike ${substring})`; + const ranByMatch = sql`exists (select 1 from ${runAttributions} ra join ${users} u on u.id = ra.user_id where ra.run_id = ${instrumentRuns.id} and (u.name ilike ${substring} or u.email ilike ${substring}))`; + + const matchCondition = or( + ilike(instrumentRuns.runId, substring), + ilike(instruments.displayName, substring), + fileMatch, + ranByMatch + ) as SQL; + + const where = and(isNull(instrumentRuns.deletedAt), matchCondition); + + // Relevance: a prefix hit on the run id (the title) outranks any substring + // hit, which outranks a metadata-only match; recency breaks ties. + const relevance = sql`case when ${instrumentRuns.runId} ilike ${prefix} then 2 when ${instrumentRuns.runId} ilike ${substring} then 1 else 0 end`; + + const rows = await db + .select({ + id: instrumentRuns.id, + runId: instrumentRuns.runId, + instrumentId: instrumentRuns.instrumentId, + instrumentName: instruments.displayName, + acquiredAt: instrumentRuns.acquiredAt, + createdAt: instrumentRuns.createdAt, + // Raw-file aggregates mirror the run list rows shown elsewhere. + fileCount: sql`cast(count(${files.id}) filter (where ${files.deletedAt} is null) as int)`, + totalSizeBytes: sql`cast(coalesce(sum(${files.sizeBytes}) filter (where ${files.deletedAt} is null), 0) as bigint)`, + matchedRunId: sql`bool_or(${instrumentRuns.runId} ilike ${substring})`, + matchedInstrument: sql`bool_or(${instruments.displayName} ilike ${substring})`, + matchedFile: sql`bool_or(${fileMatch})`, + matchedFilename: sql< + string | null + >`(select f.filename from ${files} f where f.instrument_run_id = ${instrumentRuns.id} and f.deleted_at is null and f.filename ilike ${substring} order by f.filename limit 1)`, + }) + .from(instrumentRuns) + .innerJoin(instruments, eq(instrumentRuns.instrumentId, instruments.id)) + .leftJoin( + files, + and( + eq(files.instrumentRunId, instrumentRuns.id), + eq(files.category, "raw") + ) + ) + .where(where) + .groupBy(instrumentRuns.id, instruments.displayName) + .orderBy(desc(relevance), desc(acquiredOrCreated)) + .limit(limit); + + return rows.map((row) => { + // Precedence: title match first, then a nested-file match (worth its own + // "Contains …" line), then instrument, then attribution. + let matchReason: RunMatchReason; + if (row.matchedRunId) { + matchReason = "run_id"; + } else if (row.matchedFile) { + matchReason = "file"; + } else if (row.matchedInstrument) { + matchReason = "instrument"; + } else { + matchReason = "ran_by"; + } + return { + type: "run" as const, + id: row.id, + runId: row.runId, + instrumentId: row.instrumentId, + instrumentName: row.instrumentName, + acquiredAt: row.acquiredAt ? row.acquiredAt.toISOString() : null, + createdAt: row.createdAt.toISOString(), + fileCount: row.fileCount, + totalSizeBytes: Number(row.totalSizeBytes), + matchReason, + matchedFilename: matchReason === "file" ? row.matchedFilename : null, + }; + }); +} + +async function searchFiles( + query: string, + limit: number +): Promise { + const escaped = escapeLike(query); + const substring = `%${escaped}%`; + const prefix = `${escaped}%`; + + const relevance = sql`case when ${files.filename} ilike ${prefix} then 1 else 0 end`; + + const rows = await db + .select({ + id: files.id, + filename: files.filename, + sizeBytes: files.sizeBytes, + runId: instrumentRuns.runId, + instrumentId: instrumentRuns.instrumentId, + instrumentName: instruments.displayName, + }) + .from(files) + .innerJoin(instrumentRuns, eq(files.instrumentRunId, instrumentRuns.id)) + .innerJoin(instruments, eq(instrumentRuns.instrumentId, instruments.id)) + .where( + and( + isNull(files.deletedAt), + isNull(instrumentRuns.deletedAt), + ilike(files.filename, substring) + ) + ) + .orderBy(desc(relevance), desc(files.createdAt)) + .limit(limit); + + return rows.map((row) => ({ + type: "file" as const, + id: row.id, + filename: row.filename, + sizeBytes: row.sizeBytes, + runId: row.runId, + instrumentId: row.instrumentId, + instrumentName: row.instrumentName, + })); +} + +// Instruments are few (fleet-scale), and their file patterns live in watcher +// config YAML rather than a column, so matching happens in JS over the same +// pre-aggregated list the instruments page uses. A query matches an +// instrument by display name, id, or any configured file pattern. +async function searchInstruments( + query: string, + limit: number +): Promise { + const needle = query.toLowerCase(); + const all = await getInstrumentListWithCounts(); + + const matched = all + .map((row) => { + const nameHit = + row.displayName.toLowerCase().includes(needle) || + row.id.toLowerCase().includes(needle); + const matchedPattern = row.filePatterns.find((p) => + p.toLowerCase().includes(needle) + ); + if (!(nameHit || matchedPattern)) { + return null; + } + // Name/id is the instrument's identity — rank it above a pattern-only + // match. A prefix hit on the name ranks highest. + const prefixHit = row.displayName.toLowerCase().startsWith(needle); + const relevance = nameHit ? (prefixHit ? 2 : 1) : 0; + return { + result: { + type: "instrument" as const, + id: row.id, + displayName: row.displayName, + status: row.status, + watcherStatus: getWatcherOnlineStatus(row), + lastWatcherHeartbeatAt: row.lastWatcherHeartbeatAt + ? row.lastWatcherHeartbeatAt.toISOString() + : null, + runCount: row.runCount, + matchReason: nameHit ? ("name" as const) : ("pattern" as const), + matchedPattern: nameHit ? null : (matchedPattern ?? null), + }, + relevance, + lastRunAt: row.lastRunAt?.getTime() ?? 0, + }; + }) + .filter((m): m is NonNullable => m !== null) + .sort((a, b) => b.relevance - a.relevance || b.lastRunAt - a.lastRunAt) + .slice(0, limit) + .map((m) => m.result); + + return matched; +} + +export async function globalSearch({ + query, + scope = "all", +}: { + query: string; + scope?: SearchScope; +}): Promise { + const trimmed = query.trim(); + + const empty: GlobalSearchResult = { + runs: [], + files: [], + instruments: [], + counts: { runs: 0, files: 0, instruments: 0, total: 0 }, + }; + + if (trimmed.length < MIN_QUERY_LENGTH) { + return empty; + } + + const runLimit = + scope === "all" ? ALL_TAB_PER_GROUP : scope === "runs" ? SCOPED_LIMIT : 0; + const fileLimit = + scope === "all" ? ALL_TAB_PER_GROUP : scope === "files" ? SCOPED_LIMIT : 0; + const instrumentLimit = + scope === "all" + ? ALL_TAB_PER_GROUP + : scope === "instruments" + ? SCOPED_LIMIT + : 0; + + const [runs, filesResult, instrumentsResult] = await Promise.all([ + runLimit > 0 ? searchRuns(trimmed, runLimit) : Promise.resolve([]), + fileLimit > 0 ? searchFiles(trimmed, fileLimit) : Promise.resolve([]), + instrumentLimit > 0 + ? searchInstruments(trimmed, instrumentLimit) + : Promise.resolve([]), + ]); + + return { + runs, + files: filesResult, + instruments: instrumentsResult, + counts: { + runs: runs.length, + files: filesResult.length, + instruments: instrumentsResult.length, + total: runs.length + filesResult.length + instrumentsResult.length, + }, + }; +} diff --git a/web/lib/db/schema.ts b/web/lib/db/schema.ts index 48760e3f..07dd442b 100644 --- a/web/lib/db/schema.ts +++ b/web/lib/db/schema.ts @@ -274,34 +274,46 @@ export const personalAccessTokens = pgTable( (token) => [index("idx_personal_access_tokens_user_id").on(token.userId)] ); -export const instruments = pgTable("instruments", { - // Kebab-case identifier (e.g., `spectramax-id3-plate-reader`). Also used as - // the first segment of the S3 key (`{instrument_id}/{run_id}/{filename}`). - id: text("id").primaryKey(), - // Human-readable name (e.g., "SpectraMax iD3 Plate Reader"). - displayName: text("display_name").notNull(), - // New instruments registered via the watcher CLI start as `pending` until - // confirmed by an admin. - status: instrumentStatusEnum("status").notNull().default("active"), - // Categorises the instrument for variant-specific UI (e.g., plate reader - // runs display a plate map grid). Defaults to "generic" for existing rows. - instrumentType: instrumentTypeEnum("instrument_type") - .notNull() - .default("generic"), - createdAt: timestamp("created_at", { - withTimezone: true, - mode: "date", - }) - .notNull() - .defaultNow(), - updatedAt: timestamp("updated_at", { - withTimezone: true, - mode: "date", - }) - .notNull() - .defaultNow() - .$onUpdate(() => new Date()), -}); +export const instruments = pgTable( + "instruments", + { + // Kebab-case identifier (e.g., `spectramax-id3-plate-reader`). Also used as + // the first segment of the S3 key (`{instrument_id}/{run_id}/{filename}`). + id: text("id").primaryKey(), + // Human-readable name (e.g., "SpectraMax iD3 Plate Reader"). + displayName: text("display_name").notNull(), + // New instruments registered via the watcher CLI start as `pending` until + // confirmed by an admin. + status: instrumentStatusEnum("status").notNull().default("active"), + // Categorises the instrument for variant-specific UI (e.g., plate reader + // runs display a plate map grid). Defaults to "generic" for existing rows. + instrumentType: instrumentTypeEnum("instrument_type") + .notNull() + .default("generic"), + createdAt: timestamp("created_at", { + withTimezone: true, + mode: "date", + }) + .notNull() + .defaultNow(), + updatedAt: timestamp("updated_at", { + withTimezone: true, + mode: "date", + }) + .notNull() + .defaultNow() + .$onUpdate(() => new Date()), + }, + (instrument) => [ + // Trigram GIN index backing the case-insensitive `ilike '%…%'` display-name + // match in global search. Requires the `pg_trgm` extension (created in + // migration 0029). + index("idx_instruments_display_name_trgm").using( + "gin", + sql`${instrument.displayName} gin_trgm_ops` + ), + ] +); export const watchers = pgTable( "watchers", @@ -516,6 +528,14 @@ export const instrumentRuns = pgTable( ) .where(sql`${run.deletedAt} is null`), index("idx_instrument_runs_metadata_gin").using("gin", run.metadata), + // Trigram GIN index backing the case-insensitive `ilike '%…%'` run-id + // match in global search (and the dashboard run search). Requires the + // `pg_trgm` extension (created in migration 0029). Without it these + // substring scans are sequential. + index("idx_instrument_runs_run_id_trgm").using( + "gin", + sql`${run.runId} gin_trgm_ops` + ), ] ); @@ -625,6 +645,13 @@ export const files = pgTable( sql`${file.uploadRequestedAt} is not null and ${file.uploadedAt} is null and ${file.deletedAt} is null` ), index("idx_files_metadata_gin").using("gin", file.metadata), + // Trigram GIN index backing the case-insensitive `ilike '%…%'` filename + // match used by global search and the per-run files table. Scoped to + // active rows since both callers filter out soft-deleted files. Requires + // the `pg_trgm` extension (created in migration 0029). + index("idx_files_filename_trgm") + .using("gin", sql`${file.filename} gin_trgm_ops`) + .where(sql`${file.deletedAt} is null`), ] ); diff --git a/web/lib/search-constants.ts b/web/lib/search-constants.ts new file mode 100644 index 00000000..e54ac3f7 --- /dev/null +++ b/web/lib/search-constants.ts @@ -0,0 +1,7 @@ +// Client-safe constants shared between the search UI and the server-only +// `lib/api/search` module. Kept in its own file so client components can +// import the value without pulling the DB layer into the browser bundle. + +// Below this length a query is too broad to be useful, so neither the client +// nor the backend runs a search. +export const MIN_QUERY_LENGTH = 2; diff --git a/web/scripts/reset-database.ts b/web/scripts/reset-database.ts index 7a29bded..38e1c52f 100644 --- a/web/scripts/reset-database.ts +++ b/web/scripts/reset-database.ts @@ -12,6 +12,11 @@ const db = drizzle(pool); console.log("Dropping public schema…"); await db.execute(sql`DROP SCHEMA public CASCADE`); await db.execute(sql`CREATE SCHEMA public`); +// Recreate the pg_trgm extension dropped with the public schema. The trigram +// GIN indexes in schema.ts reference `gin_trgm_ops`, so the extension must +// exist before the subsequent `db:push`. Migration 0029 handles this for the +// `db:migrate` path; push does not run migrations, hence this line. +await db.execute(sql`CREATE EXTENSION IF NOT EXISTS pg_trgm`); console.log("Database reset. Run `npm run db:push` to re-create tables."); await pool.end(); diff --git a/web/tests/integration/global-setup.ts b/web/tests/integration/global-setup.ts index 8187e510..eb4e9007 100644 --- a/web/tests/integration/global-setup.ts +++ b/web/tests/integration/global-setup.ts @@ -79,6 +79,17 @@ export async function setup() { const databaseUrl = `${PG_URL}/${TEST_DB}`; + // The trigram GIN indexes in schema.ts reference `gin_trgm_ops`, which only + // exists once the pg_trgm extension is installed. `drizzle-kit push` (below) + // does not create extensions, so ensure it exists on the fresh test DB first. + const trgmClient = new Client({ connectionString: databaseUrl }); + await trgmClient.connect(); + try { + await trgmClient.query("CREATE EXTENSION IF NOT EXISTS pg_trgm"); + } finally { + await trgmClient.end(); + } + // Stand up an in-process HTTP capture server so tests can assert on // outgoing Slack webhook calls and Slack Web API DMs without depending on // the real Slack API. diff --git a/web/tests/integration/search.test.ts b/web/tests/integration/search.test.ts new file mode 100644 index 00000000..c9bea778 --- /dev/null +++ b/web/tests/integration/search.test.ts @@ -0,0 +1,178 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { GlobalSearchResult } from "@/lib/api/search"; +import { files, instrumentRuns, instruments, watchers } from "@/lib/db/schema"; +import { + api, + closeTestDb, + getTestDb, + resetDb, + seedTestUser, +} from "@/tests/integration/helpers"; + +// Seeds a run with the given files and returns the run's UUID. +async function seedRun( + instrumentId: string, + runId: string, + filenames: string[] +): Promise { + const db = getTestDb(); + const [run] = await db + .insert(instrumentRuns) + .values({ instrumentId, runId }) + .returning({ id: instrumentRuns.id }); + if (filenames.length > 0) { + await db.insert(files).values( + filenames.map((filename, i) => ({ + instrumentRunId: run.id, + filename, + relativePath: filename, + sizeBytes: (i + 1) * 1000, + })) + ); + } + return run.id; +} + +async function search( + token: string, + query: string, + scope?: string +): Promise { + const params = new URLSearchParams({ q: query }); + if (scope) { + params.set("scope", scope); + } + const res = await api(`/api/v1/search?${params}`, { token }); + expect(res.status).toBe(200); + return (await res.json()) as GlobalSearchResult; +} + +describe("Global search API", () => { + let token: string; + + beforeAll(async () => { + await resetDb(); + ({ token } = await seedTestUser()); + + const db = getTestDb(); + await db.insert(instruments).values([ + { + id: "hina-microscope", + displayName: "Hina Microscope", + status: "active", + }, + { id: "plate-reader-x", displayName: "Plate Reader X", status: "active" }, + ]); + + // The Hina microscope watches for *.nd2; Plate Reader X watches for *.csv + // but has no runs (exercises the zero-run pattern-match case). + await db.insert(watchers).values([ + { + instrumentId: "hina-microscope", + hostname: "hina-pc", + status: "watching", + lastHeartbeatAt: new Date(), + configYaml: 'instrument:\n file_patterns:\n - "*.nd2"\n', + }, + { + instrumentId: "plate-reader-x", + hostname: "plate-pc", + status: "watching", + lastHeartbeatAt: new Date(), + configYaml: 'instrument:\n file_patterns:\n - "*.csv"\n', + }, + ]); + + await seedRun("hina-microscope", "20260706_112803_385", ["sample_012.nd2"]); + await seedRun("hina-microscope", "20260630_101502_204", ["scan_009.nd2"]); + // Literal special characters: `photo.*jpg` must match a `.*jpg` query while + // `photoXjpg` must not (the query is never treated as a regex/glob). + await seedRun("hina-microscope", "special-run", [ + "photo.*jpg", + "photoXjpg", + ]); + }); + + afterAll(async () => { + await closeTestDb(); + }); + + it("requires authentication", async () => { + const res = await api("/api/v1/search?q=nd2"); + expect(res.status).toBe(401); + }); + + it("returns an empty result below the minimum query length", async () => { + const result = await search(token, "n"); + expect(result.counts.total).toBe(0); + expect(result.runs).toHaveLength(0); + expect(result.files).toHaveLength(0); + expect(result.instruments).toHaveLength(0); + }); + + it("matches a run by its run id", async () => { + const result = await search(token, "20260706"); + const run = result.runs.find((r) => r.runId === "20260706_112803_385"); + expect(run).toBeDefined(); + expect(run?.matchReason).toBe("run_id"); + expect(run?.matchedFilename).toBeNull(); + }); + + it("surfaces runs, files, and the instrument together for a nested filename match", async () => { + const result = await search(token, "nd2"); + + // Both nd2-bearing runs surface, attributed to the contained file. + const runIds = result.runs.map((r) => r.runId); + expect(runIds).toContain("20260706_112803_385"); + expect(runIds).toContain("20260630_101502_204"); + const run = result.runs.find((r) => r.runId === "20260706_112803_385"); + expect(run?.matchReason).toBe("file"); + expect(run?.matchedFilename).toBe("sample_012.nd2"); + + // The individual files surface in the Files group. + const filenames = result.files.map((f) => f.filename); + expect(filenames).toContain("sample_012.nd2"); + expect(filenames).toContain("scan_009.nd2"); + + // The instrument surfaces via its configured *.nd2 pattern. + const instrument = result.instruments.find( + (i) => i.id === "hina-microscope" + ); + expect(instrument).toBeDefined(); + expect(instrument?.matchReason).toBe("pattern"); + expect(instrument?.matchedPattern).toBe("*.nd2"); + }); + + it("matches an instrument by display name", async () => { + const result = await search(token, "Hina"); + const instrument = result.instruments.find( + (i) => i.id === "hina-microscope" + ); + expect(instrument).toBeDefined(); + expect(instrument?.matchReason).toBe("name"); + }); + + it("scopes results to a single type when requested", async () => { + const result = await search(token, "nd2", "runs"); + expect(result.runs.length).toBeGreaterThan(0); + expect(result.files).toHaveLength(0); + expect(result.instruments).toHaveLength(0); + }); + + it("treats special characters in the query literally, not as a pattern", async () => { + const result = await search(token, ".*jpg", "files"); + const filenames = result.files.map((f) => f.filename); + expect(filenames).toContain("photo.*jpg"); + expect(filenames).not.toContain("photoXjpg"); + }); + + it("returns a pattern-matched instrument even when it has zero runs", async () => { + const result = await search(token, "csv", "instruments"); + const instrument = result.instruments.find( + (i) => i.id === "plate-reader-x" + ); + expect(instrument).toBeDefined(); + expect(instrument?.matchReason).toBe("pattern"); + expect(instrument?.runCount).toBe(0); + }); +}); From 8044d783ccfdd8ceb1d87e0643491e87922c10d3 Mon Sep 17 00:00:00 2001 From: Wasim Amiri <7220175+wasimxyz@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:24:17 -0700 Subject: [PATCH 2/3] Add MIT license across repo for open sourcing (#122) Add a root LICENSE and per-package LICENSE files, declare MIT license metadata for the lambda and shared packages (bumping their setuptools build requirement to >=77 for PEP 639), set the web package license, and document licensing and trademark in the README. Co-authored-by: Cursor --- LICENSE | 21 +++++++++++++++++++++ README.md | 8 ++++++++ lambda/LICENSE | 21 +++++++++++++++++++++ lambda/pyproject.toml | 7 ++++++- packages/shared/LICENSE | 21 +++++++++++++++++++++ packages/shared/pyproject.toml | 7 ++++++- web/package.json | 1 + 7 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 LICENSE create mode 100644 lambda/LICENSE create mode 100644 packages/shared/LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..a2d9f2bf --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Arcadia Science + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 30778588..eb0a3da8 100644 --- a/README.md +++ b/README.md @@ -82,3 +82,11 @@ make py-test-integration # Run API integration tests (requires Postgres). make fe-test-integration ``` + +## License + +Data Hub is released under the [MIT License](LICENSE). Copyright (c) 2026 Arcadia Science. + +"Data Hub" and "Arcadia Science", along with related names and logos, are marks of +Arcadia Science. The MIT License covers the source code only and does not grant any +right to use these names or logos. diff --git a/lambda/LICENSE b/lambda/LICENSE new file mode 100644 index 00000000..a2d9f2bf --- /dev/null +++ b/lambda/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Arcadia Science + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lambda/pyproject.toml b/lambda/pyproject.toml index 4cdfb0ca..a47504b1 100644 --- a/lambda/pyproject.toml +++ b/lambda/pyproject.toml @@ -2,6 +2,11 @@ name = "data-hub-lambda" version = "0.2.0" requires-python = ">=3.12" +license = "MIT" +license-files = ["LICENSE"] +authors = [ + { name = "Arcadia Science", email = "swe@arcadiascience.com" }, +] dependencies = [ "data-hub-shared", "arcadia-microscopy-tools>=0.4.1", @@ -19,7 +24,7 @@ dependencies = [ data-hub-process = "data_hub_lambda.cli:cli" [build-system] -requires = ["setuptools>=61.0", "wheel"] +requires = ["setuptools>=77.0", "wheel"] build-backend = "setuptools.build_meta" [tool.uv.sources] diff --git a/packages/shared/LICENSE b/packages/shared/LICENSE new file mode 100644 index 00000000..a2d9f2bf --- /dev/null +++ b/packages/shared/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Arcadia Science + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/shared/pyproject.toml b/packages/shared/pyproject.toml index 2bc027d2..0369cd4a 100644 --- a/packages/shared/pyproject.toml +++ b/packages/shared/pyproject.toml @@ -2,13 +2,18 @@ name = "data-hub-shared" version = "0.1.0" requires-python = ">=3.12" +license = "MIT" +license-files = ["LICENSE"] +authors = [ + { name = "Arcadia Science", email = "swe@arcadiascience.com" }, +] dependencies = [ "boto3>=1.35", "requests>=2.31", ] [build-system] -requires = ["setuptools>=61.0", "wheel"] +requires = ["setuptools>=77.0", "wheel"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] diff --git a/web/package.json b/web/package.json index 3c99b8c3..388a6974 100644 --- a/web/package.json +++ b/web/package.json @@ -3,6 +3,7 @@ "version": "0.0.1", "type": "module", "private": true, + "license": "MIT", "scripts": { "dev": "next dev --turbopack", "build": "next build", From b86c41cd6016bcd7356ff8b1ef1ca4002c2e26a2 Mon Sep 17 00:00:00 2001 From: Wasim Amiri <7220175+wasimxyz@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:15:48 -0700 Subject: [PATCH 3/3] Remove internal Arcadia-specific references for open sourcing (#123) * Remove internal Arcadia-specific references for open sourcing Prepares the repo for self-hosting by scrubbing hardcoded Arcadia resources and branding while keeping deployment/config docs intact: - Watcher: resolve staging/production API URLs via DATA_HUB_STAGING_API_URL / DATA_HUB_PRODUCTION_API_URL env vars (call-time, so ~/.data-hub/.env overrides apply), falling back to neutral example.com placeholders instead of Arcadia URLs. - Web: de-brand the dashboard metadata description; swap admin-email test fixtures to example.com. - Lambda: neutralize the api_client example URL comment. - Infra: parameterize CORS allowed origins with a note to set your own web app domain. - Docs: replace canonical Arcadia URLs with example.com placeholders and drop the Arcadia-specific environments table. Co-authored-by: Cursor * Revert watcher API URL env-var overrides Defer the self-hosted staging/production URL configuration to a follow-up. Storing the URL per environment (like watcher_ids) so switching environments doesn't re-prompt needs its own change; the env-var indirection added here was incomplete (no prompt/persistence). Restores the original API_URLS map. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- developer-docs/conventions.md | 7 ----- developer-docs/guides/installing-a-watcher.md | 2 +- developer-docs/guides/managing-tokens.md | 10 +++---- developer-docs/ops/ci-and-deployment.md | 2 +- developer-docs/reference/mcp.md | 8 +++--- infra/template.yaml | 6 ++-- lambda/src/data_hub_lambda/api_client.py | 2 +- watcher/README.md | 2 +- watcher/tests/test_run_detector.py | 2 +- web/app/page.tsx | 7 ++--- web/tests/unit/admin-emails.test.ts | 28 +++++++++---------- 11 files changed, 34 insertions(+), 42 deletions(-) diff --git a/developer-docs/conventions.md b/developer-docs/conventions.md index a157000d..9d572ea7 100644 --- a/developer-docs/conventions.md +++ b/developer-docs/conventions.md @@ -58,13 +58,6 @@ Environment-specific configuration is managed through environment variables, nev Run `make check-all` before pushing. CI enforces the same checks. -## Environments - -| Environment | API URL | S3 bucket | -| --- | --- | --- | -| Staging | `https://data-hub-env-staging-arcadia-science.vercel.app/api/v1` | `arcadia-data-hub-raw-staging` | -| Production | `https://data-hub.arcadiascience.com/api/v1` | `arcadia-data-hub-raw-production` | - ## Testing - Tests are co-located with each package: `lambda/tests/`, `watcher/tests/`, `packages/shared/tests/`. diff --git a/developer-docs/guides/installing-a-watcher.md b/developer-docs/guides/installing-a-watcher.md index 6869723e..dcd7d271 100644 --- a/developer-docs/guides/installing-a-watcher.md +++ b/developer-docs/guides/installing-a-watcher.md @@ -186,7 +186,7 @@ The watcher can't reach the Data Hub API. Check: - Your internet connection. - That the correct environment is set in the config (`staging`, `production`, or `preview`). -- That the API URL is reachable: `https://data-hub.arcadiascience.com` (production), `https://data-hub-env-staging-arcadia-science.vercel.app` (staging), or the custom URL you provided (preview). +- That the API URL is reachable. ### Files aren't being detected diff --git a/developer-docs/guides/managing-tokens.md b/developer-docs/guides/managing-tokens.md index 5abed610..63d00340 100644 --- a/developer-docs/guides/managing-tokens.md +++ b/developer-docs/guides/managing-tokens.md @@ -20,7 +20,7 @@ The plaintext token is displayed once — **copy it immediately**. It cannot be ### Via the API ```sh -curl -X POST https://data-hub.arcadiascience.com/api/v1/tokens \ +curl -X POST https://datahub.example.com/api/v1/tokens \ -H "Cookie: " \ -H "Content-Type: application/json" \ -d '{"name": "FPLC watcher", "expires_at": "2027-01-01T00:00:00Z"}' @@ -51,7 +51,7 @@ Add Data Hub to your MCP client configuration. For example, in Claude Desktop (` { "mcpServers": { "data-hub": { - "url": "https://data-hub.arcadiascience.com/api/v1/mcp", + "url": "https://datahub.example.com/api/v1/mcp", "headers": { "Authorization": "Bearer dhub_abc123..." } @@ -66,7 +66,7 @@ Or in Cursor (`.cursor/mcp.json`): { "mcpServers": { "data-hub": { - "url": "https://data-hub.arcadiascience.com/api/v1/mcp", + "url": "https://datahub.example.com/api/v1/mcp", "headers": { "Authorization": "Bearer dhub_abc123..." } @@ -82,7 +82,7 @@ See the [MCP server docs](../reference/mcp.md) for the full list of tools, resou Pass the token in the `Authorization` header: ```sh -curl https://data-hub.arcadiascience.com/api/v1/instruments \ +curl https://datahub.example.com/api/v1/instruments \ -H "Authorization: Bearer dhub_abc123..." ``` @@ -113,7 +113,7 @@ The token is immediately invalidated. Any watcher or client using it will start ### Via the API ```sh -curl -X DELETE https://data-hub.arcadiascience.com/api/v1/tokens/ \ +curl -X DELETE https://datahub.example.com/api/v1/tokens/ \ -H "Cookie: " ``` diff --git a/developer-docs/ops/ci-and-deployment.md b/developer-docs/ops/ci-and-deployment.md index b9ee304d..27af123e 100644 --- a/developer-docs/ops/ci-and-deployment.md +++ b/developer-docs/ops/ci-and-deployment.md @@ -136,7 +136,7 @@ cp infra/.env.example infra/.env.staging ``` ECR_IMAGE_URI= -DATA_HUB_API_URL=https://data-hub-env-staging-arcadia-science.vercel.app/api/v1 +DATA_HUB_API_URL=https://datahub-staging.example.com/api/v1 DATA_HUB_API_KEY= GITHUB_OIDC_PROVIDER_ARN= VERCEL_OIDC_PROVIDER_ARN= diff --git a/developer-docs/reference/mcp.md b/developer-docs/reference/mcp.md index 873295d2..4d0cfb96 100644 --- a/developer-docs/reference/mcp.md +++ b/developer-docs/reference/mcp.md @@ -20,7 +20,7 @@ Edit `claude_desktop_config.json` (macOS: `~/Library/Application Support/Claude/ { "mcpServers": { "data-hub": { - "url": "https://data-hub.arcadiascience.com/api/v1/mcp", + "url": "https://datahub.example.com/api/v1/mcp", "headers": { "Authorization": "Bearer dhub_abc123..." } @@ -39,7 +39,7 @@ Edit `.cursor/mcp.json` in your project or `~/.cursor/mcp.json` globally: { "mcpServers": { "data-hub": { - "url": "https://data-hub.arcadiascience.com/api/v1/mcp", + "url": "https://datahub.example.com/api/v1/mcp", "headers": { "Authorization": "Bearer dhub_abc123..." } @@ -54,7 +54,7 @@ Reload Cursor. Tools are invoked via the agent automatically when relevant. The endpoint follows the MCP Streamable HTTP spec, so any compliant client works. Configure it with: -- **URL**: `https://data-hub.arcadiascience.com/api/v1/mcp` +- **URL**: `https://datahub.example.com/api/v1/mcp` - **Transport**: Streamable HTTP (`GET` for the SSE stream, `POST` for client messages) - **Auth header**: `Authorization: Bearer ` @@ -146,7 +146,7 @@ The Bearer token is missing, mistyped, revoked, or expired. Verify the token at - Confirm the server is listed under `mcpServers` in the client config. - Check for JSON syntax errors in the config file. - Restart the client after editing — most clients don't hot-reload MCP server definitions. -- Hit `https://data-hub.arcadiascience.com/api/v1/mcp` with `curl -H "Authorization: Bearer "` to confirm the endpoint responds. +- Hit `https://datahub.example.com/api/v1/mcp` with `curl -H "Authorization: Bearer "` to confirm the endpoint responds. ### `get_file_download_url` vs. `get_run_archive` diff --git a/infra/template.yaml b/infra/template.yaml index ef26a206..4a456352 100644 --- a/infra/template.yaml +++ b/infra/template.yaml @@ -76,8 +76,9 @@ Resources: - AllowedMethods: - GET - HEAD + # Modify this to include your web app domain. AllowedOrigins: - - "https://data-hub.arcadiascience.com" + - "https://datahub.arcadiascience.com" - "https://*.vercel.app" - "http://localhost:3000" AllowedHeaders: @@ -191,8 +192,9 @@ Resources: - AllowedMethods: - GET - HEAD + # Modify this to include your web app domain. AllowedOrigins: - - "https://data-hub.arcadiascience.com" + - "https://datahub.arcadiascience.com" - "https://*.vercel.app" - "http://localhost:3000" AllowedHeaders: diff --git a/lambda/src/data_hub_lambda/api_client.py b/lambda/src/data_hub_lambda/api_client.py index 785011fe..04be76fb 100644 --- a/lambda/src/data_hub_lambda/api_client.py +++ b/lambda/src/data_hub_lambda/api_client.py @@ -40,7 +40,7 @@ def __init__( timeout: tuple[float, float] = DEFAULT_TIMEOUT, ) -> None: # base_url should include the API version prefix (e.g., - # "https://data-hub.arcadiascience.com/api/v1") — method paths + # "https://datahub.example.com/api/v1") — method paths # are appended relative to it. self.base_url = base_url.rstrip("/") self._timeout = timeout diff --git a/watcher/README.md b/watcher/README.md index 507d624a..c7084a78 100644 --- a/watcher/README.md +++ b/watcher/README.md @@ -1,6 +1,6 @@ # data-hub-watcher -A file-watcher agent that runs on lab instrument PCs and uploads new files to the [Arcadia Science Data Hub](https://github.com/Arcadia-Science/data-hub). It groups files into runs, retries uploads, sends heartbeats, and can optionally run as a Windows service. +A file-watcher agent that runs on lab instrument PCs and uploads new files to the [Data Hub](https://github.com/Arcadia-Science/data-hub). It groups files into runs, retries uploads, sends heartbeats, and can optionally run as a Windows service. ## Install diff --git a/watcher/tests/test_run_detector.py b/watcher/tests/test_run_detector.py index 69163ca6..72386713 100644 --- a/watcher/tests/test_run_detector.py +++ b/watcher/tests/test_run_detector.py @@ -157,7 +157,7 @@ def test_non_matching_file_returns_none(self, tmp_path: Path) -> None: class TestRunIdExtractionWindows: """Verify that backslash paths are POSIX-normalized before matching.""" - WATCH_DIR = r"D:\ArcadiaJOBS2023\JOBS_2023 Projects\Backup" + WATCH_DIR = r"D:\InstrumentData\JOBS_2023 Projects\Backup" def test_filename_prefix(self) -> None: pat = _preset_pattern("filename_prefix") diff --git a/web/app/page.tsx b/web/app/page.tsx index deb77cd9..6c7ccbeb 100644 --- a/web/app/page.tsx +++ b/web/app/page.tsx @@ -31,11 +31,8 @@ import { dashboardParamsCache, hasActiveFilters } from "@/lib/search-params"; type DashboardParams = Awaited>; -// `default: "Data Hub"` on the root metadata template already renders -// `Data Hub` here, so we skip an explicit `title` and -// override only the openGraph / twitter fields that need a strong -// `og:title` for link previews. -const description = "Instruments, runs, and watchers at Arcadia Science."; +const description = + "A central hub for your lab's instruments, runs, and files."; export const metadata: Metadata = { description, diff --git a/web/tests/unit/admin-emails.test.ts b/web/tests/unit/admin-emails.test.ts index 35d6714b..ef662777 100644 --- a/web/tests/unit/admin-emails.test.ts +++ b/web/tests/unit/admin-emails.test.ts @@ -35,40 +35,40 @@ describe("admin-emails helper", () => { }); it("parses a single email", async () => { - const { isAdminEmail } = await loadHelper("alice@arcadia.com"); - expect(isAdminEmail("alice@arcadia.com")).toBe(true); - expect(isAdminEmail("bob@arcadia.com")).toBe(false); + const { isAdminEmail } = await loadHelper("alice@example.com"); + expect(isAdminEmail("alice@example.com")).toBe(true); + expect(isAdminEmail("bob@example.com")).toBe(false); }); it("parses a comma-separated list", async () => { const { getAdminEmails, isAdminEmail } = await loadHelper( - "alice@arcadia.com,bob@arcadia.com,carol@arcadia.com" + "alice@example.com,bob@example.com,carol@example.com" ); expect(getAdminEmails().size).toBe(3); - expect(isAdminEmail("alice@arcadia.com")).toBe(true); - expect(isAdminEmail("bob@arcadia.com")).toBe(true); - expect(isAdminEmail("carol@arcadia.com")).toBe(true); + expect(isAdminEmail("alice@example.com")).toBe(true); + expect(isAdminEmail("bob@example.com")).toBe(true); + expect(isAdminEmail("carol@example.com")).toBe(true); }); it("treats comparisons as case-insensitive and trims whitespace", async () => { const { isAdminEmail } = await loadHelper( - " Alice@arcadia.com , bob@arcadia.com " + " Alice@example.com , bob@example.com " ); - expect(isAdminEmail("ALICE@arcadia.com")).toBe(true); - expect(isAdminEmail("bob@ARCADIA.com")).toBe(true); - expect(isAdminEmail(" bob@arcadia.com ")).toBe(true); - expect(isAdminEmail("carol@arcadia.com")).toBe(false); + expect(isAdminEmail("ALICE@example.com")).toBe(true); + expect(isAdminEmail("bob@EXAMPLE.com")).toBe(true); + expect(isAdminEmail(" bob@example.com ")).toBe(true); + expect(isAdminEmail("carol@example.com")).toBe(false); }); it("rejects null / undefined / empty inputs", async () => { - const { isAdminEmail } = await loadHelper("alice@arcadia.com"); + const { isAdminEmail } = await loadHelper("alice@example.com"); expect(isAdminEmail(null)).toBe(false); expect(isAdminEmail(undefined)).toBe(false); expect(isAdminEmail("")).toBe(false); }); it("ignores empty entries from leading/trailing/double commas", async () => { - const { getAdminEmails } = await loadHelper(",,alice@arcadia.com,,"); + const { getAdminEmails } = await loadHelper(",,alice@example.com,,"); expect(getAdminEmails().size).toBe(1); }); });