Add management command to re-run scanners on versions stuck

src/olympia/scanners/management/commands/retry_versions_waiting_on_scanners.py

@@ -0,0 +1,65 @@
+from datetime import datetime, timedelta
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+import olympia.core.logger
+from olympia.constants.scanners import WEBHOOK_MAX_RETRIES
+from olympia.scanners.tasks import wait_for_scanner_results
+from olympia.versions.models import Version
+
+
+log = olympia.core.logger.getLogger('z.scanners.retry_versions_waiting_on_scanners')
+
+# How long `wait_for_scanner_results` can keep retrying a version: the initial
+# countdown plus one delay per retry.
+MIN_AGE = timedelta(
+    seconds=settings.SCANNER_WEBHOOK_RETRY_DELAY * (WEBHOOK_MAX_RETRIES + 1)
+)
+
+
+class Command(BaseCommand):
+    """
+    Schedule `wait_for_scanner_results` again for the versions that are not
+    auto-approved because they are still waiting on scanner results.
+
+    Only the versions old enough for that task to be over are considered, so
+    that we don't end up with two tasks retrying the same version.
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--force',
+            action='store_true',
+            help=(
+                'Actually schedule the task. Without this flag, the command '
+                'only lists the affected versions.'
+            ),
+        )
+
+    def handle(self, *args, **options):
+        force = options['force']
+
+        qs = (
+            Version.objects.auto_approvable()
+            .filter(
+                autoapprovalsummary__is_waiting_on_scanners=True,
+                created__lt=datetime.now() - MIN_AGE,
+            )
+            .order_by('pk')
+        )
+
+        self.stdout.write(
+            f'Found {qs.count()} version(s) waiting on scanners (force={force}).'
+        )
+
+        for version in qs:
+            self.stdout.write(f'  - version {version.pk} created {version.created}')
+
+            if not force:
+                continue
+
+            wait_for_scanner_results.delay(version_pk=version.pk)
+            log.info(
+                'Scheduled wait_for_scanner_results again for version %s.', version.pk
+            )

src/olympia/scanners/tests/test_commands.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timedelta
 from io import StringIO
 from unittest import mock
 
@@ -16,6 +17,10 @@
     WEBHOOK_DURING_VALIDATION,
     WEBHOOK_ON_SOURCE_CODE_UPLOADED,
 )
+from olympia.reviewers.models import AutoApprovalSummary
+from olympia.scanners.management.commands.retry_versions_waiting_on_scanners import (
+    MIN_AGE,
+)
 from olympia.scanners.models import (
     ScannerResult,
     ScannerWebhook,
@@ -183,3 +188,91 @@ def test_continues_on_webhook_exception(self, _call_webhook_mock):
         # First one wasn't updated because the call raised.
         assert sr1.results == {'message': 'Task created'}
         assert sr2.results == {'message': 'done'}
+
+
+@mock.patch('olympia.scanners.tasks.wait_for_scanner_results.delay')
+class TestRetryVersionsWaitingOnScanners(TestCase):
+    COMMAND = 'retry_versions_waiting_on_scanners'
+
+    def _create_waiting_version(self, *, created=None, **summary_kwargs):
+        version = version_factory(
+            addon=addon_factory(), file_kw={'status': amo.STATUS_AWAITING_REVIEW}
+        )
+        version.update(created=created or self.days_ago(3))
+        summary_kwargs.setdefault('is_waiting_on_scanners', True)
+        AutoApprovalSummary.objects.create(version=version, **summary_kwargs)
+        return version
+
+    def _run(self, *args):
+        stdout = StringIO()
+        call_command(self.COMMAND, *args, stdout=stdout)
+        return stdout.getvalue()
+
+    def test_min_age(self, delay_mock):
+        # 2h countdown, then a retry every 2h, 12 times.
+        assert MIN_AGE == timedelta(hours=26)
+
+    def test_nothing_to_do(self, delay_mock):
+        output = self._run('--force')
+
+        assert 'Found 0 version(s) waiting on scanners (force=True).' in output
+        assert not delay_mock.called
+
+    def test_schedules_the_task(self, delay_mock):
+        version = self._create_waiting_version()
+
+        output = self._run('--force')
+
+        delay_mock.assert_called_once_with(version_pk=version.pk)
+        assert 'Found 1 version(s) waiting on scanners (force=True).' in output
+        assert f'version {version.pk}' in output
+
+    def test_does_nothing_without_force(self, delay_mock):
+        version = self._create_waiting_version()
+
+        output = self._run()
+
+        assert not delay_mock.called
+        assert 'Found 1 version(s) waiting on scanners (force=False).' in output
+        assert f'version {version.pk}' in output
+
+    def test_ignores_versions_not_old_enough(self, delay_mock):
+        self._create_waiting_version(created=datetime.now() - timedelta(hours=2))
+
+        self._run('--force')
+
+        assert not delay_mock.called
+
+    def test_ignores_versions_without_not_waiting_on_scanners(self, delay_mock):
+        self._create_waiting_version(is_waiting_on_scanners=False)
+
+        self._run('--force')
+
+        assert not delay_mock.called
+
+    def test_ignores_versions_without_a_summary(self, delay_mock):
+        version_factory(
+            addon=addon_factory(),
+            file_kw={'status': amo.STATUS_AWAITING_REVIEW},
+        ).update(created=self.days_ago(3))
+
+        self._run('--force')
+
+        assert not delay_mock.called
+
+    def test_ignores_versions_that_are_not_auto_approvable_anymore(self, delay_mock):
+        version = self._create_waiting_version()
+        version.file.update(status=amo.STATUS_APPROVED)
+
+        self._run('--force')
+
+        assert not delay_mock.called
+
+    def test_ignores_deleted_versions(self, delay_mock):
+        version = self._create_waiting_version()
+        version.delete()
+
+        output = self._run('--force')
+
+        assert not delay_mock.called
+        assert 'Found 0 version(s) waiting on scanners (force=True).' in output