-
-
Notifications
You must be signed in to change notification settings - Fork 14
perf(db): add hot-path indexes for sequence matching, latest-bbox and shared frame lookups #664
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
Acruve15
wants to merge
1
commit into
main
Choose a base branch
from
alexis/hot-path-indexes
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
84 changes: 84 additions & 0 deletions
84
src/migrations/versions/2026_07_30_1000-e8f3a6c9d1b7_add_hot_path_indexes.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| """add hot-path indexes for sequence matching, latest-bbox and shared frame lookups | ||
|
|
||
| Revision ID: e8f3a6c9d1b7 | ||
| Revises: c4e9f1a2b3d5 | ||
| Create Date: 2026-07-30 10:00:00.000000 | ||
|
|
||
| """ | ||
|
|
||
| from typing import Sequence, Union | ||
|
|
||
| from alembic import op | ||
| from sqlalchemy import text | ||
|
|
||
| # revision identifiers, used by Alembic. | ||
| revision: str = "e8f3a6c9d1b7" | ||
| down_revision: Union[str, None] = "c4e9f1a2b3d5" | ||
|
Acruve15 marked this conversation as resolved.
|
||
| branch_labels: Union[str, Sequence[str], None] = None | ||
|
Acruve15 marked this conversation as resolved.
|
||
| depends_on: Union[str, Sequence[str], None] = None | ||
|
Acruve15 marked this conversation as resolved.
|
||
|
|
||
| # (index name, table, columns). Kept in sync with the __table_args__ declarations in | ||
| # app.models, which is what puts these indexes in create_all-built test databases too. | ||
| INDEXES = ( | ||
| ("ix_sequences_camera_pose_last_seen", "sequences", ["camera_id", "pose_id", "last_seen_at"]), | ||
| ("ix_detections_sequence_id_created_at", "detections", ["sequence_id", "created_at"]), | ||
| ("ix_detections_bucket_key", "detections", ["bucket_key"]), | ||
| ) | ||
|
|
||
|
|
||
| def _drop_if_invalid(index_name: str, table_name: str) -> None: | ||
| """Drop an index left INVALID by a cancelled CONCURRENTLY build. | ||
|
|
||
| Without this the migration is not safely re-runnable: if_not_exists sees the leftover | ||
| relation and skips creating it, the revision stamps, and the upgrade reports success while | ||
| the planner ignores the invalid index, silently leaving these queries on sequential scans. | ||
|
|
||
| The lookup joins pg_class by name rather than casting the name to regclass, since that cast | ||
| raises when the relation does not exist yet (the common case) instead of returning no rows. | ||
| """ | ||
| row = ( | ||
| op | ||
| .get_bind() | ||
| .execute( | ||
| text( | ||
| "SELECT idx.indisvalid FROM pg_index idx " | ||
| "JOIN pg_class c ON c.oid = idx.indexrelid " | ||
| "WHERE c.relname = :name" | ||
| ), | ||
| {"name": index_name}, | ||
| ) | ||
| .first() | ||
| ) | ||
| if row is not None and not row[0]: | ||
| op.drop_index(index_name, table_name=table_name, if_exists=True, postgresql_concurrently=True) | ||
|
|
||
|
|
||
| def upgrade() -> None: | ||
| # Three query shapes on the detection hot path had no index backing them: | ||
| # - a pose's recently-seen sequences (camera_id, pose_id, last_seen_at), run on every | ||
| # POST /detections during spatial matching | ||
| # - the latest real bbox of a sequence (sequence_id, created_at), run once per candidate | ||
| # sequence per detection, and also the shape the player's sequence reads sort on | ||
| # - sibling rows sharing a frame object (bucket_key), on DELETE /detections/{id} | ||
| # | ||
| # detections is the highest-write table, so a plain CREATE INDEX would hold ACCESS EXCLUSIVE | ||
| # against camera ingest for the whole build. CONCURRENTLY cannot run inside a transaction and | ||
| # env.py wraps the migration run in one, hence the autocommit block. | ||
| with op.get_context().autocommit_block(): | ||
| for index_name, table_name, columns in INDEXES: | ||
| _drop_if_invalid(index_name, table_name) | ||
| op.create_index( | ||
| index_name, | ||
| table_name, | ||
| columns, | ||
| unique=False, | ||
| if_not_exists=True, | ||
| postgresql_concurrently=True, | ||
| ) | ||
|
|
||
|
|
||
| def downgrade() -> None: | ||
| # DROP INDEX CONCURRENTLY is likewise non-transactional. | ||
| with op.get_context().autocommit_block(): | ||
| for index_name, table_name, _ in reversed(INDEXES): | ||
| op.drop_index(index_name, table_name=table_name, if_exists=True, postgresql_concurrently=True) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| import pytest | ||
| from sqlmodel import SQLModel, text | ||
| from sqlmodel.ext.asyncio.session import AsyncSession | ||
|
|
||
| # Hot-path indexes, declared in two places that must not drift: the alembic migration (what | ||
| # production runs) and __table_args__ in app.models (what create_all gives a test database built | ||
| # without migrations). Drift is invisible at runtime, it just makes one environment plan queries | ||
| # differently from the other, so both sides are pinned against this list. | ||
| EXPECTED_INDEXES = { | ||
| "detections": {"ix_detections_sequence_id_created_at", "ix_detections_bucket_key"}, | ||
| "sequences": {"ix_sequences_camera_pose_last_seen"}, | ||
| } | ||
|
|
||
|
|
||
| @pytest.mark.parametrize(("table", "expected"), EXPECTED_INDEXES.items()) | ||
| def test_hot_path_indexes_are_declared_on_the_models(table: str, expected: set): | ||
| """Guards the model side. A DB assertion cannot cover this: the test database is migrated, | ||
| so the indexes are present whether or not __table_args__ still declares them.""" | ||
| declared = {index.name for index in SQLModel.metadata.tables[table].indexes} | ||
| assert expected <= declared, f"not declared on {table}: {sorted(expected - declared)}" | ||
|
|
||
|
|
||
| @pytest.mark.asyncio | ||
| @pytest.mark.parametrize(("table", "expected"), EXPECTED_INDEXES.items()) | ||
| async def test_hot_path_indexes_exist_in_the_database(async_session: AsyncSession, table: str, expected: set): | ||
| """Guards the migration side: a fresh database must end up with all of them.""" | ||
| stmt = text("SELECT indexname FROM pg_indexes WHERE tablename = :table").bindparams(table=table) | ||
| present = {row[0] for row in (await async_session.exec(stmt)).all()} | ||
| assert expected <= present, f"missing on {table}: {sorted(expected - present)}" |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.