From bcdc6c7becab42f697759fff582c5e00a968acda Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Mon, 21 Sep 2026 10:25:33 +0200 Subject: [PATCH 1/5] feat: fix slow tiles --- specifyweb/backend/barvis/tests/__init__.py | 0 specifyweb/backend/barvis/tests/test_views.py | 51 ++++++++++++ specifyweb/backend/barvis/views.py | 78 +++++++------------ .../0049_determination_current_taxon_index.py | 18 +++++ specifyweb/specify/models.py | 1 + 5 files changed, 100 insertions(+), 48 deletions(-) create mode 100644 specifyweb/backend/barvis/tests/__init__.py create mode 100644 specifyweb/backend/barvis/tests/test_views.py create mode 100644 specifyweb/specify/migrations/0049_determination_current_taxon_index.py diff --git a/specifyweb/backend/barvis/tests/__init__.py b/specifyweb/backend/barvis/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/specifyweb/backend/barvis/tests/test_views.py b/specifyweb/backend/barvis/tests/test_views.py new file mode 100644 index 00000000000..84b917eb191 --- /dev/null +++ b/specifyweb/backend/barvis/tests/test_views.py @@ -0,0 +1,51 @@ +from specifyweb.specify.models import Determination, Taxon, Taxontreedefitem +from specifyweb.specify.tests.test_api import ApiTests + +from specifyweb.backend.barvis.views import get_taxon_bar_data + + +class TaxonBarTests(ApiTests): + def setUp(self): + super().setUp() + self.discipline.taxontreedef = self.taxontreedef + self.discipline.save() + + root_rank = Taxontreedefitem.objects.create( + name='Root', rankid=0, treedef=self.taxontreedef + ) + species_rank = Taxontreedefitem.objects.create( + name='Species', rankid=220, parent=root_rank, treedef=self.taxontreedef + ) + self.root = Taxon.objects.create( + name='Root', rankid=0, definition=self.taxontreedef, + definitionitem=root_rank, + ) + self.taxon = Taxon.objects.create( + name='Test species', rankid=220, parent=self.root, + definition=self.taxontreedef, definitionitem=species_rank, + ) + + def test_returns_current_determination_counts_without_changing_rows(self): + Determination.objects.create( + collectionobject=self.collectionobjects[0], taxon=self.taxon, + collectionmemberid=self.collection.id, iscurrent=True, + createdbyagent=self.agent, modifiedbyagent=self.agent, + ) + Determination.objects.create( + collectionobject=self.collectionobjects[1], taxon=self.taxon, + collectionmemberid=self.collection.id, iscurrent=True, + createdbyagent=self.agent, modifiedbyagent=self.agent, + ) + Determination.objects.create( + collectionobject=self.collectionobjects[2], taxon=self.taxon, + collectionmemberid=self.collection.id, iscurrent=False, + createdbyagent=self.agent, modifiedbyagent=self.agent, + ) + + self.assertCountEqual( + get_taxon_bar_data(self.collection), + [ + (self.root.id, 0, None, 'Root', 0), + (self.taxon.id, 220, self.root.id, 'Test species', 2), + ], + ) diff --git a/specifyweb/backend/barvis/views.py b/specifyweb/backend/barvis/views.py index bff43890db9..e0fadc459ba 100644 --- a/specifyweb/backend/barvis/views.py +++ b/specifyweb/backend/barvis/views.py @@ -1,60 +1,42 @@ +from django.db.models import Count, IntegerField, OuterRef, Subquery +from django.db.models.functions import Coalesce from django.http import HttpResponse -from django.db.models import Count, Q from specifyweb.middleware.general import require_GET -from specifyweb.specify.views import login_maybe_required from specifyweb.specify.api.filter_by_col import filter_by_collection from specifyweb.specify.api.serializers import toJson -from specifyweb.specify.models import Taxon +from specifyweb.specify.models import Determination, Taxon +from specifyweb.specify.views import login_maybe_required + -from django.db import connection +def get_taxon_bar_data(collection): + """Return taxon tile rows with a correlated current-count lookup.""" + current_determination_counts = ( + Determination.objects + .filter(taxon_id=OuterRef('pk'), iscurrent=True) + .values('taxon_id') + .annotate(count=Count('id')) + .values('count') + ) + taxons = Taxon.objects.annotate( + current_determination_count=Coalesce( + Subquery( + current_determination_counts, + output_field=IntegerField(), + ), + 0, + ) + ).values_list( + 'id', 'rankid', 'parent_id', 'name', 'current_determination_count' + ) + return list(filter_by_collection(taxons, collection)) @require_GET @login_maybe_required def taxon_bar(request): - # "Returns the data for creating a taxon tiles visualization." - # cursor = connection.cursor() - # cursor.execute(""" - # SELECT t.TaxonID, - # t.RankID, - # t.ParentID, - # t.Name, - # (SELECT COUNT(*) FROM determination d WHERE t.TaxonID = d.TaxonID AND d.IsCurrent = 1) - # FROM taxon t - # WHERE t.TaxonTreeDefID = %s - # """, [request.specify_collection.discipline.taxontreedef_id]) - - # Implementing the previous SQL query in Django ORM: - taxons = ( - Taxon.objects.annotate( - current_determination_count=Count( - 'determinations', filter=Q(determinations__iscurrent=True)) - ) - .values_list("id", "rankid", "parent_id", "name", "current_determination_count") + """Return the data for creating a taxon tiles visualization.""" + return HttpResponse( + toJson(get_taxon_bar_data(request.specify_collection)), + content_type='application/json', ) - filtered_taxons = filter_by_collection(taxons, request.specify_collection) - result = toJson(list(filtered_taxons)) - - # SELECT d.TaxonID, COUNT(DISTINCT d.CollectionObjectID), t.ParentID - # FROM determination d - # INNER JOIN taxon t ON t.TaxonID = d.TaxonID - # WHERE d.CollectionMemberID = %s - # AND d.IsCurrent = 1 - # GROUP BY d.TaxonId - # ORDER BY d.TaxonId - # """, [request.specify_collection.id]) - # result = toJson(cursor.fetchall()) - # session = Session() - # query = session.query( - # Determination.TaxonID, - # func.count(distinct(Determination.CollectionObjectID)), - # Taxon.ParentID) \ - # .join(Taxon, Determination.TaxonID == Taxon.taxonId) \ - # .filter(Determination.collectionMemberId == request.specify_collection.id) \ - # .filter(Determination.isCurrent == True) \ - # .group_by(Determination.TaxonID).order_by(Determination.TaxonID) - - # result = toJson(list(query)) - # session.close() - return HttpResponse(result, content_type='application/json') diff --git a/specifyweb/specify/migrations/0049_determination_current_taxon_index.py b/specifyweb/specify/migrations/0049_determination_current_taxon_index.py new file mode 100644 index 00000000000..817b51d6534 --- /dev/null +++ b/specifyweb/specify/migrations/0049_determination_current_taxon_index.py @@ -0,0 +1,18 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('specify', '0048_taxontreedefitem_parent_context_delete'), + ] + + operations = [ + migrations.AddIndex( + model_name='determination', + index=models.Index( + fields=['iscurrent', 'taxon'], + name='DetCurrentTaxonIDX', + ), + ), + ] diff --git a/specifyweb/specify/models.py b/specifyweb/specify/models.py index 6ab316a15f8..09c64c1cb86 100644 --- a/specifyweb/specify/models.py +++ b/specifyweb/specify/models.py @@ -2888,6 +2888,7 @@ class Meta: indexes = [ models.Index(fields=['determineddate'], name='DeterminedDateIDX'), models.Index(fields=['collectionmemberid'], name='DetMemIDX'), + models.Index(fields=['iscurrent', 'taxon'], name='DetCurrentTaxonIDX'), models.Index(fields=['alternatename'], name='AlterNameIDX'), models.Index(fields=['guid'], name='DeterminationGuidIDX'), models.Index(fields=['typestatusname'], name='TypeStatusNameIDX') From af36f50ab218bb0e01b91a52c0e47f773cfc23d3 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:22:41 +0200 Subject: [PATCH 2/5] feat: add index for fetching current objects in the collection --- specifyweb/backend/barvis/tests/test_views.py | 13 ++++++++++++- specifyweb/backend/barvis/views.py | 6 +++++- .../0049_determination_current_taxon_index.py | 7 +++++++ specifyweb/specify/models.py | 4 ++++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/specifyweb/backend/barvis/tests/test_views.py b/specifyweb/backend/barvis/tests/test_views.py index 84b917eb191..1203f9b0444 100644 --- a/specifyweb/backend/barvis/tests/test_views.py +++ b/specifyweb/backend/barvis/tests/test_views.py @@ -1,4 +1,4 @@ -from specifyweb.specify.models import Determination, Taxon, Taxontreedefitem +from specifyweb.specify.models import Collection, Determination, Taxon, Taxontreedefitem from specifyweb.specify.tests.test_api import ApiTests from specifyweb.backend.barvis.views import get_taxon_bar_data @@ -26,6 +26,12 @@ def setUp(self): ) def test_returns_current_determination_counts_without_changing_rows(self): + collection_2 = Collection.objects.create( + catalognumformatname='test', + collectionname='TestCollection2', + isembeddedcollectingevent=False, + discipline=self.discipline, + ) Determination.objects.create( collectionobject=self.collectionobjects[0], taxon=self.taxon, collectionmemberid=self.collection.id, iscurrent=True, @@ -41,6 +47,11 @@ def test_returns_current_determination_counts_without_changing_rows(self): collectionmemberid=self.collection.id, iscurrent=False, createdbyagent=self.agent, modifiedbyagent=self.agent, ) + Determination.objects.create( + collectionobject=self.collectionobjects[2], taxon=self.taxon, + collectionmemberid=collection_2.id, iscurrent=True, + createdbyagent=self.agent, modifiedbyagent=self.agent, + ) self.assertCountEqual( get_taxon_bar_data(self.collection), diff --git a/specifyweb/backend/barvis/views.py b/specifyweb/backend/barvis/views.py index e0fadc459ba..d87e595d595 100644 --- a/specifyweb/backend/barvis/views.py +++ b/specifyweb/backend/barvis/views.py @@ -13,7 +13,11 @@ def get_taxon_bar_data(collection): """Return taxon tile rows with a correlated current-count lookup.""" current_determination_counts = ( Determination.objects - .filter(taxon_id=OuterRef('pk'), iscurrent=True) + .filter( + taxon_id=OuterRef('pk'), + collectionmemberid=collection.id, + iscurrent=True, + ) .values('taxon_id') .annotate(count=Count('id')) .values('count') diff --git a/specifyweb/specify/migrations/0049_determination_current_taxon_index.py b/specifyweb/specify/migrations/0049_determination_current_taxon_index.py index 817b51d6534..790b72c128d 100644 --- a/specifyweb/specify/migrations/0049_determination_current_taxon_index.py +++ b/specifyweb/specify/migrations/0049_determination_current_taxon_index.py @@ -15,4 +15,11 @@ class Migration(migrations.Migration): name='DetCurrentTaxonIDX', ), ), + migrations.AddIndex( + model_name='determination', + index=models.Index( + fields=['collectionmemberid', 'iscurrent', 'taxon'], + name='DetColMemCurrentTaxonIDX', + ), + ), ] diff --git a/specifyweb/specify/models.py b/specifyweb/specify/models.py index 09c64c1cb86..2fd47af196d 100644 --- a/specifyweb/specify/models.py +++ b/specifyweb/specify/models.py @@ -2889,6 +2889,10 @@ class Meta: models.Index(fields=['determineddate'], name='DeterminedDateIDX'), models.Index(fields=['collectionmemberid'], name='DetMemIDX'), models.Index(fields=['iscurrent', 'taxon'], name='DetCurrentTaxonIDX'), + models.Index( + fields=['collectionmemberid', 'iscurrent', 'taxon'], + name='DetColMemCurrentTaxonIDX', + ), models.Index(fields=['alternatename'], name='AlterNameIDX'), models.Index(fields=['guid'], name='DeterminationGuidIDX'), models.Index(fields=['typestatusname'], name='TypeStatusNameIDX') From e37db2ca2fd8d60c031ef06788325f552e86a758 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:31:29 +0200 Subject: [PATCH 3/5] test: fix taxon tree def --- specifyweb/backend/barvis/tests/test_views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specifyweb/backend/barvis/tests/test_views.py b/specifyweb/backend/barvis/tests/test_views.py index 1203f9b0444..36db0d2b505 100644 --- a/specifyweb/backend/barvis/tests/test_views.py +++ b/specifyweb/backend/barvis/tests/test_views.py @@ -9,6 +9,8 @@ def setUp(self): super().setUp() self.discipline.taxontreedef = self.taxontreedef self.discipline.save() + self.taxontreedef.discipline = self.discipline + self.taxontreedef.save() root_rank = Taxontreedefitem.objects.create( name='Root', rankid=0, treedef=self.taxontreedef From 6fd2e8e32a344d00dea9f6cb06bfceba9969a412 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:49:30 +0200 Subject: [PATCH 4/5] fix: remove unnecessary index --- .../migrations/0049_determination_current_taxon_index.py | 7 ------- specifyweb/specify/models.py | 1 - 2 files changed, 8 deletions(-) diff --git a/specifyweb/specify/migrations/0049_determination_current_taxon_index.py b/specifyweb/specify/migrations/0049_determination_current_taxon_index.py index 790b72c128d..4e93f1aa6d0 100644 --- a/specifyweb/specify/migrations/0049_determination_current_taxon_index.py +++ b/specifyweb/specify/migrations/0049_determination_current_taxon_index.py @@ -8,13 +8,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AddIndex( - model_name='determination', - index=models.Index( - fields=['iscurrent', 'taxon'], - name='DetCurrentTaxonIDX', - ), - ), migrations.AddIndex( model_name='determination', index=models.Index( diff --git a/specifyweb/specify/models.py b/specifyweb/specify/models.py index 2fd47af196d..f479befb0c3 100644 --- a/specifyweb/specify/models.py +++ b/specifyweb/specify/models.py @@ -2888,7 +2888,6 @@ class Meta: indexes = [ models.Index(fields=['determineddate'], name='DeterminedDateIDX'), models.Index(fields=['collectionmemberid'], name='DetMemIDX'), - models.Index(fields=['iscurrent', 'taxon'], name='DetCurrentTaxonIDX'), models.Index( fields=['collectionmemberid', 'iscurrent', 'taxon'], name='DetColMemCurrentTaxonIDX', From 17f7ab798e8a398ca4eadb880df41c40472af3e8 Mon Sep 17 00:00:00 2001 From: Grant Fitzsimmons <37256050+grantfitzsimmons@users.noreply.github.com> Date: Sat, 26 Sep 2026 17:36:41 +0200 Subject: [PATCH 5/5] fix(taxon-tiles): prevent multiple rows per CO --- specifyweb/backend/barvis/tests/test_views.py | 7 +++++++ specifyweb/backend/barvis/views.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/specifyweb/backend/barvis/tests/test_views.py b/specifyweb/backend/barvis/tests/test_views.py index 36db0d2b505..be63f485a33 100644 --- a/specifyweb/backend/barvis/tests/test_views.py +++ b/specifyweb/backend/barvis/tests/test_views.py @@ -44,6 +44,13 @@ def test_returns_current_determination_counts_without_changing_rows(self): collectionmemberid=self.collection.id, iscurrent=True, createdbyagent=self.agent, modifiedbyagent=self.agent, ) + Determination.objects.bulk_create([ + Determination( + collectionobject=self.collectionobjects[0], taxon=self.taxon, + collectionmemberid=self.collection.id, iscurrent=True, + createdbyagent=self.agent, modifiedbyagent=self.agent, + ), + ]) Determination.objects.create( collectionobject=self.collectionobjects[2], taxon=self.taxon, collectionmemberid=self.collection.id, iscurrent=False, diff --git a/specifyweb/backend/barvis/views.py b/specifyweb/backend/barvis/views.py index d87e595d595..3e37f749ae9 100644 --- a/specifyweb/backend/barvis/views.py +++ b/specifyweb/backend/barvis/views.py @@ -19,7 +19,7 @@ def get_taxon_bar_data(collection): iscurrent=True, ) .values('taxon_id') - .annotate(count=Count('id')) + .annotate(count=Count('collectionobject', distinct=True)) .values('count') ) taxons = Taxon.objects.annotate(