From 731a367653cfa3c43ecd4e4de8711e3ab5986775 Mon Sep 17 00:00:00 2001 From: Valentin Rigal Date: Thu, 21 Nov 2024 17:49:19 +0100 Subject: [PATCH 01/11] Base endpoint to list issues --- backend/code_review_backend/issues/api.py | 139 +++++++++++++++++++--- 1 file changed, 123 insertions(+), 16 deletions(-) diff --git a/backend/code_review_backend/issues/api.py b/backend/code_review_backend/issues/api.py index 173d4b53f..37cac4707 100644 --- a/backend/code_review_backend/issues/api.py +++ b/backend/code_review_backend/issues/api.py @@ -6,11 +6,20 @@ from datetime import date, datetime, timedelta from django.conf import settings -from django.db.models import BooleanField, Count, ExpressionWrapper, Prefetch, Q +from django.db.models import ( + BooleanField, + Count, + Exists, + ExpressionWrapper, + OuterRef, + Prefetch, + Q, +) from django.db.models.functions import TruncDate from django.shortcuts import get_object_or_404 from django.urls import path from django.utils.decorators import method_decorator +from django.utils.functional import cached_property from django.views.decorators.cache import cache_page from rest_framework import generics, mixins, routers, status, viewsets from rest_framework.exceptions import APIException, ValidationError @@ -486,6 +495,90 @@ def get_queryset(self): return qs.filter(**filters).order_by("created").distinct() +class GenericIssueList(generics.ListAPIView): + serializer_class = IssueHashSerializer + allowed_modes = ["unresolved", "known", "closed"] + + @cached_property + def diff(self): + return get_object_or_404( + Diff.objects.select_related("revision"), id=self.kwargs["diff_id"] + ) + + @cached_property + def mode(self): + if (mode := self.kwargs["mode"]) not in self.allowed_modes: + raise APIException( + detail="mode argument must be one of {self.allowed_modes}" + ) + return mode + + def get_queryset(self): + return getattr(self, f"get_{self.mode}")() + + def distinct_issues(self, qs): + """ + Convert a list of issue links to unique couples of (issue_id, issue_hash) + """ + attributes = ("issue_id", "issue__hash") + return qs.order_by(*attributes).values_list(*attributes).distinct() + + def get_unresolved(self): + """ + Issues that were linked to a previous diff of the same + parent revision, and are still present on the current diff. + """ + return self.distinct_issues( + self.diff.revision.issue_links.annotate( + unresolved=Exists( + IssueLink.objects.exclude(diff=self.diff).filter( + revision=self.diff.revision, + issue=OuterRef("issue"), + ) + ) + ).filter(unresolved=True) + ) + + def get_known(self): + """ + Issues that have also being detected from mozilla-central. + """ + return self.distinct_issues( + self.diff.issue_links.annotate( + known=Exists( + IssueLink.objects.filter( + revision__base_repository__slug="mozilla-central", + issue=OuterRef("issue"), + ) + ) + ).filter(known=True) + ) + + def get_closed(self): + """ + Issues that were listed on the previous diff of the same + parent revision, but does not exist on the current diff. + """ + # Retrieve the previous diff + previous_diff = ( + self.diff.revision.diffs.filter(created__lt=self.diff.created) + .order_by("created") + .last() + ) + if not previous_diff: + return [] + return self.distinct_issues( + previous_diff.issue_links.annotate( + closed=Exists( + IssueLink.objects.filter( + diff=self.diff, + issue=OuterRef("issue"), + ) + ) + ).filter(closed=True) + ) + + # Build exposed urls for the API router = routers.DefaultRouter() router.register(r"repository", RepositoryViewSet) @@ -497,18 +590,32 @@ def get_queryset(self): ) router.register(r"diff", DiffViewSet, basename="diffs") router.register(r"diff/(?P\d+)/issues", IssueViewSet, basename="issues") -urls = router.urls + [ - path( - "revision//issues/", - IssueBulkCreate.as_view(), - name="revision-issues-bulk", - ), - path("check/stats/", IssueCheckStats.as_view(), name="issue-checks-stats"), - path("check/history/", IssueCheckHistory.as_view(), name="issue-checks-history"), - path( - "check////", - IssueCheckDetails.as_view(), - name="issue-check-details", - ), - path("issues//", IssueList.as_view(), name="repository-issues"), -] + +urls = ( + [ + # Prevails on the generic issues endpoint + path( + "diff//issues//", + GenericIssueList.as_view(), + name="issue-list", + ), + ] + + router.urls + + [ + path( + "revision//issues/", + IssueBulkCreate.as_view(), + name="revision-issues-bulk", + ), + path("check/stats/", IssueCheckStats.as_view(), name="issue-checks-stats"), + path( + "check/history/", IssueCheckHistory.as_view(), name="issue-checks-history" + ), + path( + "check////", + IssueCheckDetails.as_view(), + name="issue-check-details", + ), + path("issues//", IssueList.as_view(), name="repository-issues"), + ] +) From dff247d0b10654a28a00a48e1c224c8882ff8704 Mon Sep 17 00:00:00 2001 From: Valentin Rigal Date: Fri, 22 Nov 2024 11:35:47 +0100 Subject: [PATCH 02/11] Move new endpoint to API /v2/ --- backend/code_review_backend/app/urls.py | 3 +- backend/code_review_backend/issues/api_v2.py | 107 +++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 backend/code_review_backend/issues/api_v2.py diff --git a/backend/code_review_backend/app/urls.py b/backend/code_review_backend/app/urls.py index f6f4cd8c8..e0e5ae5ef 100644 --- a/backend/code_review_backend/app/urls.py +++ b/backend/code_review_backend/app/urls.py @@ -10,7 +10,7 @@ from drf_yasg.views import get_schema_view from rest_framework import permissions -from code_review_backend.issues import api +from code_review_backend.issues import api, api_v2 # Build Swagger schema view schema_view = get_schema_view( @@ -28,6 +28,7 @@ urlpatterns = [ path("", lambda request: redirect("docs/", permanent=False)), path("v1/", include(api.urls)), + path("v2/", include(api_v2.urls)), path("admin/", admin.site.urls), path( r"docs\.json|\.yaml)", diff --git a/backend/code_review_backend/issues/api_v2.py b/backend/code_review_backend/issues/api_v2.py new file mode 100644 index 000000000..bd955bfe5 --- /dev/null +++ b/backend/code_review_backend/issues/api_v2.py @@ -0,0 +1,107 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from django.db.models import Exists, OuterRef +from django.shortcuts import get_object_or_404 +from django.urls import path +from django.utils.functional import cached_property +from rest_framework.exceptions import ValidationError +from rest_framework.generics import ListAPIView + +from code_review_backend.issues.models import Diff, IssueLink +from code_review_backend.issues.serializers import IssueHashSerializer + + +class IssueList(ListAPIView): + serializer_class = IssueHashSerializer + allowed_modes = ["unresolved", "known", "closed"] + + @cached_property + def diff(self): + return get_object_or_404( + Diff.objects.select_related("revision"), id=self.kwargs["diff_id"] + ) + + @cached_property + def mode(self): + if (mode := self.kwargs["mode"]) not in self.allowed_modes: + raise ValidationError( + detail=f"mode argument must be one of {self.allowed_modes}" + ) + return mode + + def get_queryset(self): + return getattr(self, f"get_{self.mode}")() + + def distinct_issues(self, qs): + """ + Convert a list of issue links to unique couples of (issue_id, issue_hash) + """ + attributes = ("issue_id", "issue__hash") + return qs.order_by(*attributes).values_list(*attributes).distinct() + + def get_unresolved(self): + """ + Issues that were linked to a previous diff of the same + parent revision, and are still present on the current diff. + """ + return self.distinct_issues( + self.diff.revision.issue_links.annotate( + unresolved=Exists( + IssueLink.objects.exclude(diff=self.diff).filter( + revision=self.diff.revision, + issue=OuterRef("issue"), + ) + ) + ).filter(unresolved=True) + ) + + def get_known(self): + """ + Issues that have also being detected from mozilla-central. + """ + return self.distinct_issues( + self.diff.issue_links.annotate( + known=Exists( + IssueLink.objects.filter( + revision__base_repository__slug="mozilla-central", + issue=OuterRef("issue"), + ) + ) + ).filter(known=True) + ) + + def get_closed(self): + """ + Issues that were listed on the previous diff of the same + parent revision, but does not exist on the current diff. + """ + # Retrieve the previous diff + previous_diff = ( + self.diff.revision.diffs.filter(created__lt=self.diff.created) + .order_by("created") + .last() + ) + if not previous_diff: + return [] + return self.distinct_issues( + previous_diff.issue_links.annotate( + closed=Exists( + IssueLink.objects.filter( + diff=self.diff, + issue=OuterRef("issue"), + ) + ) + ).filter(closed=True) + ) + + +urls = [ + # Prevails on the generic issues endpoint + path( + "diff//issues//", + IssueList.as_view(), + name="issue-list", + ), +] From 11d2b59552b66c99acdf2d69f80a70e375185093 Mon Sep 17 00:00:00 2001 From: Valentin Rigal Date: Fri, 22 Nov 2024 12:06:02 +0100 Subject: [PATCH 03/11] Add test cases --- .../issues/tests/test_api_v2.py | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 backend/code_review_backend/issues/tests/test_api_v2.py diff --git a/backend/code_review_backend/issues/tests/test_api_v2.py b/backend/code_review_backend/issues/tests/test_api_v2.py new file mode 100644 index 000000000..56a5738c6 --- /dev/null +++ b/backend/code_review_backend/issues/tests/test_api_v2.py @@ -0,0 +1,151 @@ +from rest_framework import status +from rest_framework.test import APITestCase + +from code_review_backend.issues.models import Issue, Repository + + +class CreationAPIv2TestCase(APITestCase): + def setUp(self): + # Create the main repository + self.repo_main = Repository.objects.create( + slug="mozilla-central", url="https://hg.mozilla.org/mozilla-central" + ) + self.repo_try = Repository.objects.create( + slug="myrepo-try", url="http://repo.test/try" + ) + + self.issues = Issue.objects.bulk_create( + [ + Issue(**attrs) + for attrs in [ + {"hash": "0" * 32}, + {"hash": "1" * 32}, + {"hash": "2" * 32}, + {"hash": "3" * 32}, + ] + ] + ) + + # Create historic revision with some issues + self.revision_main = self.repo_main.head_revisions.create( + phabricator_id=456, + phabricator_phid="PHID-REV-XXX", + title="Main revision", + base_repository=self.repo_main, + ) + self.revision_main.issue_links.create(issue=self.issues[0]) + self.revision_main.issue_links.create(issue=self.issues[1]) + + # Create a new revions, with two successive diffs + self.revision_last = self.repo_try.head_revisions.create( + phabricator_id=1337, + phabricator_phid="PHID-REV-YYY", + title="Bug XXX - Yet another bug", + base_repository=self.repo_try, + ) + self.diff1 = self.revision_last.diffs.create( + id=1, + repository=self.repo_try, + review_task_id="deadbeef123", + phid="PHID-DIFF-xxx", + ) + self.diff1.issue_links.create(revision=self.revision_last, issue=self.issues[0]) + self.diff1.issue_links.create(revision=self.revision_last, issue=self.issues[2]) + self.diff1.issue_links.create(revision=self.revision_last, issue=self.issues[3]) + + self.diff2 = self.revision_last.diffs.create( + id=2, + repository=self.repo_try, + review_task_id="coffee12345", + phid="PHID-DIFF-yyy", + ) + self.diff2.issue_links.create(revision=self.revision_last, issue=self.issues[0]) + self.diff2.issue_links.create(revision=self.revision_last, issue=self.issues[2]) + + def test_list_issues_invalid_mode(self): + with self.assertNumQueries(0): + response = self.client.get( + f"/v2/diff/{self.diff2.id}/issues/test/", format="json" + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertListEqual( + response.json(), + ["mode argument must be one of ['unresolved', 'known', 'closed']"], + ) + + def test_list_issues_invalid_diff(self): + with self.assertNumQueries(1): + response = self.client.get("/v2/diff/424242/issues/known/") + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_list_issues_known(self): + with self.assertNumQueries(3): + response = self.client.get(f"/v2/diff/{self.diff2.id}/issues/known/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertCountEqual( + response.json()["results"], + [{"id": str(self.issues[0].id), "hash": self.issues[0].hash}], + ) + + def test_list_issues_unresolved_first_diff(self): + """No issue can be marked as unresolved on a new diff""" + with self.assertNumQueries(2): + response = self.client.get(f"/v2/diff/{self.diff1.id}/issues/unresolved/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertCountEqual( + response.json()["results"], + [], + ) + + def test_list_issues_unresolved(self): + with self.assertNumQueries(4): + response = self.client.get(f"/v2/diff/{self.diff2.id}/issues/unresolved/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertCountEqual( + response.json()["results"], + [{"id": str(self.issues[2].id), "hash": self.issues[2].hash}], + ) + + def test_list_issues_closed_first_diff(self): + with self.assertNumQueries(2): + response = self.client.get(f"/v2/diff/{self.diff1.id}/issues/closed/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertCountEqual(response.json()["results"], []) + + def test_list_issues_closed(self): + with self.assertNumQueries(4): + response = self.client.get(f"/v2/diff/{self.diff2.id}/issues/closed/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertListEqual( + response.json()["results"], + [{"id": str(self.issues[3].id), "hash": self.issues[3].hash}], + ) + + def test_list_issues_new_diff_reopen(self): + """ + Open a new diff with 1 known issue and one that has been closed in the previous diff. + Both are listed as new, and the two issues of the previous diff listed as closed. + """ + diff = self.revision_last.diffs.create( + id=3, + repository=self.repo_try, + review_task_id="42", + phid="PHID-DIFF-42", + ) + new_issue = Issue.objects.create(hash="4" * 32) + diff.issue_links.create(revision=self.revision_last, issue=new_issue) + diff.issue_links.create(revision=self.revision_last, issue=self.issues[3]) + self.assertCountEqual( + self.client.get(f"/v2/diff/{diff.id}/issues/unresolved/").json()["results"], + [], + ) + self.assertCountEqual( + self.client.get(f"/v2/diff/{diff.id}/issues/known/").json()["results"], [] + ) + self.assertCountEqual( + self.client.get(f"/v2/diff/{diff.id}/issues/closed/").json()["results"], + [ + {"id": str(self.issues[0].id), "hash": self.issues[0].hash}, + {"id": str(self.issues[2].id), "hash": self.issues[2].hash}, + ], + ) From ff1520500b798b1dae4af1fd9a97599c89d53f8c Mon Sep 17 00:00:00 2001 From: Valentin Rigal Date: Fri, 22 Nov 2024 17:09:42 +0100 Subject: [PATCH 04/11] Use a specific serializer --- backend/code_review_backend/issues/api_v2.py | 6 +++--- .../code_review_backend/issues/serializers_v2.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 backend/code_review_backend/issues/serializers_v2.py diff --git a/backend/code_review_backend/issues/api_v2.py b/backend/code_review_backend/issues/api_v2.py index bd955bfe5..bd6b37ca3 100644 --- a/backend/code_review_backend/issues/api_v2.py +++ b/backend/code_review_backend/issues/api_v2.py @@ -10,11 +10,11 @@ from rest_framework.generics import ListAPIView from code_review_backend.issues.models import Diff, IssueLink -from code_review_backend.issues.serializers import IssueHashSerializer +from code_review_backend.issues.serializers_v2 import IssueLinkHashSerializer class IssueList(ListAPIView): - serializer_class = IssueHashSerializer + serializer_class = IssueLinkHashSerializer allowed_modes = ["unresolved", "known", "closed"] @cached_property @@ -39,7 +39,7 @@ def distinct_issues(self, qs): Convert a list of issue links to unique couples of (issue_id, issue_hash) """ attributes = ("issue_id", "issue__hash") - return qs.order_by(*attributes).values_list(*attributes).distinct() + return qs.order_by(*attributes).values(*attributes).distinct() def get_unresolved(self): """ diff --git a/backend/code_review_backend/issues/serializers_v2.py b/backend/code_review_backend/issues/serializers_v2.py new file mode 100644 index 000000000..6252d65e1 --- /dev/null +++ b/backend/code_review_backend/issues/serializers_v2.py @@ -0,0 +1,15 @@ +from rest_framework import serializers + +from code_review_backend.issues.models import IssueLink + + +class IssueLinkHashSerializer(serializers.ModelSerializer): + """Serialize an Issue hash from the IssueLink M2M""" + + id = serializers.UUIDField(source="issue_id") + hash = serializers.CharField(source="issue__hash", max_length=32) + + class Meta: + model = IssueLink + fields = ("id", "hash") + read_only_fields = ("id", "hash") From f843d3d7d2e2db1995fabc2f8072b54e24e6322c Mon Sep 17 00:00:00 2001 From: Valentin Rigal Date: Fri, 22 Nov 2024 17:40:44 +0100 Subject: [PATCH 05/11] Update endpoint --- backend/code_review_backend/issues/api_v2.py | 54 ++++++++++++-------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/backend/code_review_backend/issues/api_v2.py b/backend/code_review_backend/issues/api_v2.py index bd6b37ca3..bfe0d7ffb 100644 --- a/backend/code_review_backend/issues/api_v2.py +++ b/backend/code_review_backend/issues/api_v2.py @@ -2,7 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from django.db.models import Exists, OuterRef +from django.conf import settings +from django.db.models import Exists, OuterRef, Value from django.shortcuts import get_object_or_404 from django.urls import path from django.utils.functional import cached_property @@ -32,47 +33,60 @@ def mode(self): return mode def get_queryset(self): - return getattr(self, f"get_{self.mode}")() + return getattr(self, f"{self.mode}_issues").filter(**{self.mode: True}) def distinct_issues(self, qs): """ Convert a list of issue links to unique couples of (issue_id, issue_hash) """ attributes = ("issue_id", "issue__hash") + if "postgresql" in settings.DATABASES["default"]["ENGINE"]: + return qs.order_by(*attributes).values(*attributes).distinct(*attributes) return qs.order_by(*attributes).values(*attributes).distinct() - def get_unresolved(self): + @property + def known_issues(self): """ - Issues that were linked to a previous diff of the same - parent revision, and are still present on the current diff. + Issues that have also being detected from mozilla-central. """ return self.distinct_issues( - self.diff.revision.issue_links.annotate( - unresolved=Exists( - IssueLink.objects.exclude(diff=self.diff).filter( - revision=self.diff.revision, + self.diff.issue_links.annotate( + known=Exists( + IssueLink.objects.filter( + revision__base_repository__slug="mozilla-central", issue=OuterRef("issue"), ) ) - ).filter(unresolved=True) + ) ) - def get_known(self): + @property + def unresolved_issues(self): """ - Issues that have also being detected from mozilla-central. + Issues that were linked to the previous diff of the same + parent revision, and are still present on the current diff. + Filters out issues that are known by the backend. """ + previous_diff = ( + self.diff.revision.diffs.filter(created__lt=self.diff.created) + .order_by("created") + .last() + ) + if not previous_diff: + return IssueLink.objects.none().annotate(unresolved=Value("True")) return self.distinct_issues( - self.diff.issue_links.annotate( - known=Exists( + self.known_issues.filter(known=False).annotate( + unresolved=Exists( IssueLink.objects.filter( - revision__base_repository__slug="mozilla-central", + diff=previous_diff, issue=OuterRef("issue"), ) ) - ).filter(known=True) + ) ) - def get_closed(self): + @property + def closed_issues(self): """ Issues that were listed on the previous diff of the same parent revision, but does not exist on the current diff. @@ -84,16 +98,16 @@ def get_closed(self): .last() ) if not previous_diff: - return [] + return IssueLink.objects.none().annotate(closed=Value("True")) return self.distinct_issues( previous_diff.issue_links.annotate( - closed=Exists( + closed=~Exists( IssueLink.objects.filter( diff=self.diff, issue=OuterRef("issue"), ) ) - ).filter(closed=True) + ) ) From 816f7edf665d2689be79e02f43fe3a5d41a3c286 Mon Sep 17 00:00:00 2001 From: Valentin Rigal Date: Tue, 26 Nov 2024 12:56:55 +0100 Subject: [PATCH 06/11] Remove old endpoint from api_v1 --- backend/code_review_backend/issues/api.py | 139 +++------------------- 1 file changed, 16 insertions(+), 123 deletions(-) diff --git a/backend/code_review_backend/issues/api.py b/backend/code_review_backend/issues/api.py index 37cac4707..173d4b53f 100644 --- a/backend/code_review_backend/issues/api.py +++ b/backend/code_review_backend/issues/api.py @@ -6,20 +6,11 @@ from datetime import date, datetime, timedelta from django.conf import settings -from django.db.models import ( - BooleanField, - Count, - Exists, - ExpressionWrapper, - OuterRef, - Prefetch, - Q, -) +from django.db.models import BooleanField, Count, ExpressionWrapper, Prefetch, Q from django.db.models.functions import TruncDate from django.shortcuts import get_object_or_404 from django.urls import path from django.utils.decorators import method_decorator -from django.utils.functional import cached_property from django.views.decorators.cache import cache_page from rest_framework import generics, mixins, routers, status, viewsets from rest_framework.exceptions import APIException, ValidationError @@ -495,90 +486,6 @@ def get_queryset(self): return qs.filter(**filters).order_by("created").distinct() -class GenericIssueList(generics.ListAPIView): - serializer_class = IssueHashSerializer - allowed_modes = ["unresolved", "known", "closed"] - - @cached_property - def diff(self): - return get_object_or_404( - Diff.objects.select_related("revision"), id=self.kwargs["diff_id"] - ) - - @cached_property - def mode(self): - if (mode := self.kwargs["mode"]) not in self.allowed_modes: - raise APIException( - detail="mode argument must be one of {self.allowed_modes}" - ) - return mode - - def get_queryset(self): - return getattr(self, f"get_{self.mode}")() - - def distinct_issues(self, qs): - """ - Convert a list of issue links to unique couples of (issue_id, issue_hash) - """ - attributes = ("issue_id", "issue__hash") - return qs.order_by(*attributes).values_list(*attributes).distinct() - - def get_unresolved(self): - """ - Issues that were linked to a previous diff of the same - parent revision, and are still present on the current diff. - """ - return self.distinct_issues( - self.diff.revision.issue_links.annotate( - unresolved=Exists( - IssueLink.objects.exclude(diff=self.diff).filter( - revision=self.diff.revision, - issue=OuterRef("issue"), - ) - ) - ).filter(unresolved=True) - ) - - def get_known(self): - """ - Issues that have also being detected from mozilla-central. - """ - return self.distinct_issues( - self.diff.issue_links.annotate( - known=Exists( - IssueLink.objects.filter( - revision__base_repository__slug="mozilla-central", - issue=OuterRef("issue"), - ) - ) - ).filter(known=True) - ) - - def get_closed(self): - """ - Issues that were listed on the previous diff of the same - parent revision, but does not exist on the current diff. - """ - # Retrieve the previous diff - previous_diff = ( - self.diff.revision.diffs.filter(created__lt=self.diff.created) - .order_by("created") - .last() - ) - if not previous_diff: - return [] - return self.distinct_issues( - previous_diff.issue_links.annotate( - closed=Exists( - IssueLink.objects.filter( - diff=self.diff, - issue=OuterRef("issue"), - ) - ) - ).filter(closed=True) - ) - - # Build exposed urls for the API router = routers.DefaultRouter() router.register(r"repository", RepositoryViewSet) @@ -590,32 +497,18 @@ def get_closed(self): ) router.register(r"diff", DiffViewSet, basename="diffs") router.register(r"diff/(?P\d+)/issues", IssueViewSet, basename="issues") - -urls = ( - [ - # Prevails on the generic issues endpoint - path( - "diff//issues//", - GenericIssueList.as_view(), - name="issue-list", - ), - ] - + router.urls - + [ - path( - "revision//issues/", - IssueBulkCreate.as_view(), - name="revision-issues-bulk", - ), - path("check/stats/", IssueCheckStats.as_view(), name="issue-checks-stats"), - path( - "check/history/", IssueCheckHistory.as_view(), name="issue-checks-history" - ), - path( - "check////", - IssueCheckDetails.as_view(), - name="issue-check-details", - ), - path("issues//", IssueList.as_view(), name="repository-issues"), - ] -) +urls = router.urls + [ + path( + "revision//issues/", + IssueBulkCreate.as_view(), + name="revision-issues-bulk", + ), + path("check/stats/", IssueCheckStats.as_view(), name="issue-checks-stats"), + path("check/history/", IssueCheckHistory.as_view(), name="issue-checks-history"), + path( + "check////", + IssueCheckDetails.as_view(), + name="issue-check-details", + ), + path("issues//", IssueList.as_view(), name="repository-issues"), +] From 3b13119b6e143ff02f43741068109f3076cff4be Mon Sep 17 00:00:00 2001 From: Valentin Rigal Date: Fri, 6 Dec 2024 16:19:20 +0100 Subject: [PATCH 07/11] Move everything to code_review_backend.issues.v2 --- backend/code_review_backend/app/urls.py | 3 ++- backend/code_review_backend/issues/v2/__init__.py | 0 backend/code_review_backend/issues/{api_v2.py => v2/api.py} | 2 +- .../issues/{serializers_v2.py => v2/serializers.py} | 0 backend/code_review_backend/issues/v2/tests/__init__.py | 0 .../{tests/test_api_v2.py => v2/tests/test_list_issues.py} | 2 +- 6 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 backend/code_review_backend/issues/v2/__init__.py rename backend/code_review_backend/issues/{api_v2.py => v2/api.py} (98%) rename backend/code_review_backend/issues/{serializers_v2.py => v2/serializers.py} (100%) create mode 100644 backend/code_review_backend/issues/v2/tests/__init__.py rename backend/code_review_backend/issues/{tests/test_api_v2.py => v2/tests/test_list_issues.py} (99%) diff --git a/backend/code_review_backend/app/urls.py b/backend/code_review_backend/app/urls.py index e0e5ae5ef..b7862ab28 100644 --- a/backend/code_review_backend/app/urls.py +++ b/backend/code_review_backend/app/urls.py @@ -10,7 +10,8 @@ from drf_yasg.views import get_schema_view from rest_framework import permissions -from code_review_backend.issues import api, api_v2 +from code_review_backend.issues import api +from code_review_backend.issues.v2 import api as api_v2 # Build Swagger schema view schema_view = get_schema_view( diff --git a/backend/code_review_backend/issues/v2/__init__.py b/backend/code_review_backend/issues/v2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/code_review_backend/issues/api_v2.py b/backend/code_review_backend/issues/v2/api.py similarity index 98% rename from backend/code_review_backend/issues/api_v2.py rename to backend/code_review_backend/issues/v2/api.py index bfe0d7ffb..e5a191a4e 100644 --- a/backend/code_review_backend/issues/api_v2.py +++ b/backend/code_review_backend/issues/v2/api.py @@ -11,7 +11,7 @@ from rest_framework.generics import ListAPIView from code_review_backend.issues.models import Diff, IssueLink -from code_review_backend.issues.serializers_v2 import IssueLinkHashSerializer +from code_review_backend.issues.v2.serializers import IssueLinkHashSerializer class IssueList(ListAPIView): diff --git a/backend/code_review_backend/issues/serializers_v2.py b/backend/code_review_backend/issues/v2/serializers.py similarity index 100% rename from backend/code_review_backend/issues/serializers_v2.py rename to backend/code_review_backend/issues/v2/serializers.py diff --git a/backend/code_review_backend/issues/v2/tests/__init__.py b/backend/code_review_backend/issues/v2/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/code_review_backend/issues/tests/test_api_v2.py b/backend/code_review_backend/issues/v2/tests/test_list_issues.py similarity index 99% rename from backend/code_review_backend/issues/tests/test_api_v2.py rename to backend/code_review_backend/issues/v2/tests/test_list_issues.py index 56a5738c6..54dd84064 100644 --- a/backend/code_review_backend/issues/tests/test_api_v2.py +++ b/backend/code_review_backend/issues/v2/tests/test_list_issues.py @@ -4,7 +4,7 @@ from code_review_backend.issues.models import Issue, Repository -class CreationAPIv2TestCase(APITestCase): +class ListIssuesTestCase(APITestCase): def setUp(self): # Create the main repository self.repo_main = Repository.objects.create( From 9e5f280ae91f1524fe95af5392b082a364aa381b Mon Sep 17 00:00:00 2001 From: Valentin Rigal Date: Fri, 6 Dec 2024 16:30:07 +0100 Subject: [PATCH 08/11] Suggestions --- backend/code_review_backend/issues/v2/api.py | 42 ++++++++----------- .../issues/v2/serializers.py | 5 +-- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/backend/code_review_backend/issues/v2/api.py b/backend/code_review_backend/issues/v2/api.py index e5a191a4e..6713f0cf0 100644 --- a/backend/code_review_backend/issues/v2/api.py +++ b/backend/code_review_backend/issues/v2/api.py @@ -2,8 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from django.conf import settings -from django.db.models import Exists, OuterRef, Value +from django.core.exceptions import BadRequest +from django.db.models import Exists, OuterRef from django.shortcuts import get_object_or_404 from django.urls import path from django.utils.functional import cached_property @@ -24,6 +24,16 @@ def diff(self): Diff.objects.select_related("revision"), id=self.kwargs["diff_id"] ) + @cached_property + def previous_diff(self): + previous_diff = ( + self.diff.revision.diffs.filter(created__lt=self.diff.created) + .order_by("created") + .last() + ) + if not previous_diff: + raise BadRequest("No previous diff was found to compare issues") + @cached_property def mode(self): if (mode := self.kwargs["mode"]) not in self.allowed_modes: @@ -40,20 +50,18 @@ def distinct_issues(self, qs): Convert a list of issue links to unique couples of (issue_id, issue_hash) """ attributes = ("issue_id", "issue__hash") - if "postgresql" in settings.DATABASES["default"]["ENGINE"]: - return qs.order_by(*attributes).values(*attributes).distinct(*attributes) - return qs.order_by(*attributes).values(*attributes).distinct() + return qs.order_by(*attributes).values(*attributes).distinct(*attributes) @property def known_issues(self): """ - Issues that have also being detected from mozilla-central. + Issues that have also being detected from the base repository (e.g. mozilla-central). """ return self.distinct_issues( self.diff.issue_links.annotate( known=Exists( IssueLink.objects.filter( - revision__base_repository__slug="mozilla-central", + revision__base_repository_id=self.diff.revision.base_repository_id, issue=OuterRef("issue"), ) ) @@ -67,18 +75,11 @@ def unresolved_issues(self): parent revision, and are still present on the current diff. Filters out issues that are known by the backend. """ - previous_diff = ( - self.diff.revision.diffs.filter(created__lt=self.diff.created) - .order_by("created") - .last() - ) - if not previous_diff: - return IssueLink.objects.none().annotate(unresolved=Value("True")) return self.distinct_issues( self.known_issues.filter(known=False).annotate( unresolved=Exists( IssueLink.objects.filter( - diff=previous_diff, + diff=self.previous_diff, issue=OuterRef("issue"), ) ) @@ -91,16 +92,8 @@ def closed_issues(self): Issues that were listed on the previous diff of the same parent revision, but does not exist on the current diff. """ - # Retrieve the previous diff - previous_diff = ( - self.diff.revision.diffs.filter(created__lt=self.diff.created) - .order_by("created") - .last() - ) - if not previous_diff: - return IssueLink.objects.none().annotate(closed=Value("True")) return self.distinct_issues( - previous_diff.issue_links.annotate( + self.previous_diff.issue_links.annotate( closed=~Exists( IssueLink.objects.filter( diff=self.diff, @@ -112,7 +105,6 @@ def closed_issues(self): urls = [ - # Prevails on the generic issues endpoint path( "diff//issues//", IssueList.as_view(), diff --git a/backend/code_review_backend/issues/v2/serializers.py b/backend/code_review_backend/issues/v2/serializers.py index 6252d65e1..e5088fa30 100644 --- a/backend/code_review_backend/issues/v2/serializers.py +++ b/backend/code_review_backend/issues/v2/serializers.py @@ -6,10 +6,9 @@ class IssueLinkHashSerializer(serializers.ModelSerializer): """Serialize an Issue hash from the IssueLink M2M""" - id = serializers.UUIDField(source="issue_id") - hash = serializers.CharField(source="issue__hash", max_length=32) + id = serializers.UUIDField(source="issue_id", read_only=True) + hash = serializers.CharField(source="issue__hash", max_length=32, read_only=True) class Meta: model = IssueLink fields = ("id", "hash") - read_only_fields = ("id", "hash") From 6b7943a8381bd1632cb2aa857cf7b9c3502db6f9 Mon Sep 17 00:00:00 2001 From: Valentin Rigal Date: Fri, 6 Dec 2024 16:51:22 +0100 Subject: [PATCH 09/11] Serialize the previous diff ID --- backend/code_review_backend/issues/v2/api.py | 48 ++++++++++++------- .../issues/v2/serializers.py | 5 ++ 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/backend/code_review_backend/issues/v2/api.py b/backend/code_review_backend/issues/v2/api.py index 6713f0cf0..77cda0d67 100644 --- a/backend/code_review_backend/issues/v2/api.py +++ b/backend/code_review_backend/issues/v2/api.py @@ -11,12 +11,33 @@ from rest_framework.generics import ListAPIView from code_review_backend.issues.models import Diff, IssueLink -from code_review_backend.issues.v2.serializers import IssueLinkHashSerializer +from code_review_backend.issues.v2.serializers import DiffIssuesSerializer class IssueList(ListAPIView): - serializer_class = IssueLinkHashSerializer + """ + Returns issues that are linked to a diff, and depending on the selected mode: + * known: Issues that have also being detected from the base repository (e.g. mozilla-central). + `previous_diff_id` is always returned as null when listing known issues. + * unresolved: Issues that were linked to the previous diff of the same parent revision, and are + still present on the current diff. Filters out issues that are known by the backend. + * closed: Issues that were listed on the previous diff of the same parent revision, but does + not exist on the current diff. + + This endpoint does not uses pagination. + """ + allowed_modes = ["unresolved", "known", "closed"] + pagination_class = None + + def get_serializer(self, queryset, many=True): + return DiffIssuesSerializer( + { + "previous_diff_id": self.previous_diff and self.previous_diff.id, + "issues": queryset, + }, + context=self.get_serializer_context(), + ) @cached_property def diff(self): @@ -26,13 +47,14 @@ def diff(self): @cached_property def previous_diff(self): - previous_diff = ( + if self.mode == "known": + # No need to search for a previous diff for known issues + return None + return ( self.diff.revision.diffs.filter(created__lt=self.diff.created) .order_by("created") .last() ) - if not previous_diff: - raise BadRequest("No previous diff was found to compare issues") @cached_property def mode(self): @@ -54,9 +76,6 @@ def distinct_issues(self, qs): @property def known_issues(self): - """ - Issues that have also being detected from the base repository (e.g. mozilla-central). - """ return self.distinct_issues( self.diff.issue_links.annotate( known=Exists( @@ -70,11 +89,8 @@ def known_issues(self): @property def unresolved_issues(self): - """ - Issues that were linked to the previous diff of the same - parent revision, and are still present on the current diff. - Filters out issues that are known by the backend. - """ + if not self.previous_diff: + raise BadRequest("No previous diff was found to compare issues") return self.distinct_issues( self.known_issues.filter(known=False).annotate( unresolved=Exists( @@ -88,10 +104,8 @@ def unresolved_issues(self): @property def closed_issues(self): - """ - Issues that were listed on the previous diff of the same - parent revision, but does not exist on the current diff. - """ + if not self.previous_diff: + raise BadRequest("No previous diff was found to compare issues") return self.distinct_issues( self.previous_diff.issue_links.annotate( closed=~Exists( diff --git a/backend/code_review_backend/issues/v2/serializers.py b/backend/code_review_backend/issues/v2/serializers.py index e5088fa30..c384c4efa 100644 --- a/backend/code_review_backend/issues/v2/serializers.py +++ b/backend/code_review_backend/issues/v2/serializers.py @@ -12,3 +12,8 @@ class IssueLinkHashSerializer(serializers.ModelSerializer): class Meta: model = IssueLink fields = ("id", "hash") + + +class DiffIssuesSerializer(serializers.Serializer): + previous_diff_id = serializers.UUIDField(allow_null=True, read_only=True) + issues = IssueLinkHashSerializer(many=True) From c935df5900b5833a32ac845b1df975db2019aa9d Mon Sep 17 00:00:00 2001 From: Valentin Rigal Date: Fri, 6 Dec 2024 17:55:48 +0100 Subject: [PATCH 10/11] Compare head_repository to detect known issues --- backend/code_review_backend/issues/v2/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/code_review_backend/issues/v2/api.py b/backend/code_review_backend/issues/v2/api.py index 77cda0d67..8cee8f27c 100644 --- a/backend/code_review_backend/issues/v2/api.py +++ b/backend/code_review_backend/issues/v2/api.py @@ -80,7 +80,7 @@ def known_issues(self): self.diff.issue_links.annotate( known=Exists( IssueLink.objects.filter( - revision__base_repository_id=self.diff.revision.base_repository_id, + revision__head_repository_id=self.diff.revision.base_repository_id, issue=OuterRef("issue"), ) ) From 700f7090ec3355bb443bab11151a0e6266ceefe2 Mon Sep 17 00:00:00 2001 From: Valentin Rigal Date: Fri, 6 Dec 2024 17:56:36 +0100 Subject: [PATCH 11/11] Update tests --- .../issues/v2/tests/test_list_issues.py | 92 ++++++++++++------- 1 file changed, 58 insertions(+), 34 deletions(-) diff --git a/backend/code_review_backend/issues/v2/tests/test_list_issues.py b/backend/code_review_backend/issues/v2/tests/test_list_issues.py index 54dd84064..fc0939268 100644 --- a/backend/code_review_backend/issues/v2/tests/test_list_issues.py +++ b/backend/code_review_backend/issues/v2/tests/test_list_issues.py @@ -32,6 +32,7 @@ def setUp(self): phabricator_phid="PHID-REV-XXX", title="Main revision", base_repository=self.repo_main, + head_repository=self.repo_main, ) self.revision_main.issue_links.create(issue=self.issues[0]) self.revision_main.issue_links.create(issue=self.issues[1]) @@ -41,7 +42,8 @@ def setUp(self): phabricator_id=1337, phabricator_phid="PHID-REV-YYY", title="Bug XXX - Yet another bug", - base_repository=self.repo_try, + base_repository=self.repo_main, + head_repository=self.repo_try, ) self.diff1 = self.revision_last.diffs.create( id=1, @@ -79,46 +81,58 @@ def test_list_issues_invalid_diff(self): self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) def test_list_issues_known(self): - with self.assertNumQueries(3): + with self.assertNumQueries(2): response = self.client.get(f"/v2/diff/{self.diff2.id}/issues/known/") self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertCountEqual( - response.json()["results"], - [{"id": str(self.issues[0].id), "hash": self.issues[0].hash}], + self.maxDiff = None + self.assertDictEqual( + response.json(), + { + "previous_diff_id": None, + "issues": [ + {"id": str(self.issues[0].id), "hash": self.issues[0].hash}, + ], + }, ) def test_list_issues_unresolved_first_diff(self): """No issue can be marked as unresolved on a new diff""" with self.assertNumQueries(2): - response = self.client.get(f"/v2/diff/{self.diff1.id}/issues/unresolved/") - self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertCountEqual( - response.json()["results"], - [], - ) + response = self.client.get( + f"/v2/diff/{self.diff1.id}/issues/unresolved/", format="json" + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) def test_list_issues_unresolved(self): - with self.assertNumQueries(4): + with self.assertNumQueries(3): response = self.client.get(f"/v2/diff/{self.diff2.id}/issues/unresolved/") self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertCountEqual( - response.json()["results"], - [{"id": str(self.issues[2].id), "hash": self.issues[2].hash}], + self.assertDictEqual( + response.json(), + { + "previous_diff_id": "1", + "issues": [ + {"id": str(self.issues[2].id), "hash": self.issues[2].hash}, + ], + }, ) def test_list_issues_closed_first_diff(self): + """No issue can be marked as closed on a new diff""" with self.assertNumQueries(2): response = self.client.get(f"/v2/diff/{self.diff1.id}/issues/closed/") - self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertCountEqual(response.json()["results"], []) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) def test_list_issues_closed(self): - with self.assertNumQueries(4): + with self.assertNumQueries(3): response = self.client.get(f"/v2/diff/{self.diff2.id}/issues/closed/") self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertListEqual( - response.json()["results"], - [{"id": str(self.issues[3].id), "hash": self.issues[3].hash}], + self.assertDictEqual( + response.json(), + { + "previous_diff_id": "1", + "issues": [{"id": str(self.issues[3].id), "hash": self.issues[3].hash}], + }, ) def test_list_issues_new_diff_reopen(self): @@ -135,17 +149,27 @@ def test_list_issues_new_diff_reopen(self): new_issue = Issue.objects.create(hash="4" * 32) diff.issue_links.create(revision=self.revision_last, issue=new_issue) diff.issue_links.create(revision=self.revision_last, issue=self.issues[3]) - self.assertCountEqual( - self.client.get(f"/v2/diff/{diff.id}/issues/unresolved/").json()["results"], - [], - ) - self.assertCountEqual( - self.client.get(f"/v2/diff/{diff.id}/issues/known/").json()["results"], [] - ) - self.assertCountEqual( - self.client.get(f"/v2/diff/{diff.id}/issues/closed/").json()["results"], - [ - {"id": str(self.issues[0].id), "hash": self.issues[0].hash}, - {"id": str(self.issues[2].id), "hash": self.issues[2].hash}, - ], + self.assertDictEqual( + self.client.get(f"/v2/diff/{diff.id}/issues/unresolved/").json(), + { + "previous_diff_id": "2", + "issues": [], + }, + ) + self.assertDictEqual( + self.client.get(f"/v2/diff/{diff.id}/issues/known/").json(), + { + "previous_diff_id": None, + "issues": [], + }, + ) + self.assertDictEqual( + self.client.get(f"/v2/diff/{diff.id}/issues/closed/").json(), + { + "previous_diff_id": "2", + "issues": [ + {"id": str(self.issues[0].id), "hash": self.issues[0].hash}, + {"id": str(self.issues[2].id), "hash": self.issues[2].hash}, + ], + }, )