Add the on_version_scanned scanner webhook event

docs/topics/development/scanner_pipeline.md

@@ -132,6 +132,12 @@ When some results are still missing two hours after the version was created, the
 with the same payload, and keeps doing so every two hours, up to 12 times. The
 last retry therefore happens 24 hours after the version was created.
 
+AMO waits in two steps, each with a budget of its own: first for the results
+[`on_version_scanned`](#scanner-on-version-scanned) aggregates, then for
+`on_version_scanned` itself, which cannot be sent before they are all in. A
+scanner listening to `on_version_scanned` therefore gets the same 24 hours,
+counted from the moment the event was sent to it.
+
 Once the retries are exhausted, or when the payload can no longer be rebuilt
 (e.g., the uploaded file a `during_validation` payload points to is gone), AMO
 records artificial results matching the special `SCANNER_RESULTS_MISSING`
@@ -480,6 +486,70 @@ The payload sent looks like this:
 }
 ```
 
+(scanner-on-version-scanned)=
+### `on_version_scanned`
+
+This event occurs once **every** scanner listening to a predefined set of events
+has returned its results for the version. It is meant for scanners that need to
+make a decision informed by what the other scanners found.
+
+In addition to the usual `addon` and `version` properties, the payload contains a
+`scanner_results` object. It has one property per aggregated event, each mapping
+the name of a scanner listening to that event to the results it returned. A
+`null` value means that scanner skipped the event.
+
+A scanner listening to both aggregated events appears under both, so its two sets
+of results remain distinguishable.
+
+```json
+{
+  "addon": {
+    // Similar to the `on_version_created` event.
+  },
+  "version": {
+    // Similar to the `on_version_created` event.
+  },
+  "scanner_results": {
+    "during_validation": {
+      "some-scanner": {
+        "version": "1.0",
+        "matchedRules": ["SOME_RULE"],
+        "annotations": {}
+      }
+    },
+    "on_version_created": {
+      "some-scanner": {
+        "version": "1.0",
+        "matchedRules": [],
+        "annotations": {}
+      },
+      "some-other-scanner": null
+    }
+  },
+  "event": "on_version_scanned",
+  "scanner_result_url": "http://olympia.test/api/v5/scanner/results/127/"
+}
+```
+
+```{warning}
+`scanner_results` reports every result AMO holds for an aggregated event, which
+is not the same set as the scanners currently subscribed to it. A scanner that
+subscribed *after* the version was created is missing entirely, because AMO
+never called it; conversely, a scanner whose webhook has since been deactivated
+is still reported, because its results had already been collected. Don't assume
+the object lists every scanner currently subscribed; look up the scanners you
+care about by name and handle their absence.
+```
+
+This event blocks auto-approval like the events it aggregates, so the version is
+not auto-approved until the scanners listening to it have answered as well. When
+AMO gave up on one of the aggregated scanners, the event is still sent, with the
+`SCANNER_RESULTS_MISSING` [rule](#scanner-rules) in place of that scanner's
+results.
+
+Results [pushed](#scanner-push) by a scanner are not part of the payload, and
+pushing results does not trigger this event.
+
 (scanner-push)=
 ### `push`
 

src/olympia/constants/scanners.py

@@ -81,21 +81,30 @@
 WEBHOOK_ON_VERSION_CREATED = 3
 WEBHOOK_PUSH = 4
 WEBHOOK_ON_LISTING_CHANGED = 5
+WEBHOOK_ON_VERSION_SCANNED = 6
 
 WEBHOOK_EVENTS = {
     WEBHOOK_DURING_VALIDATION: 'during_validation',
     WEBHOOK_ON_SOURCE_CODE_UPLOADED: 'on_source_code_uploaded',
     WEBHOOK_ON_VERSION_CREATED: 'on_version_created',
     WEBHOOK_PUSH: 'push',
     WEBHOOK_ON_LISTING_CHANGED: 'on_listing_changed',
+    WEBHOOK_ON_VERSION_SCANNED: 'on_version_scanned',
 }
 
-# Events we wait on before auto-approving a version.
-WEBHOOK_EVENTS_BLOCKING_AUTO_APPROVAL = [
+# The events whose results are aggregated into the `on_version_scanned`
+# payload. We wait for all of them before sending it.
+WEBHOOK_EVENTS_AGGREGATED_BY_ON_VERSION_SCANNED = [
     WEBHOOK_DURING_VALIDATION,
     WEBHOOK_ON_VERSION_CREATED,
 ]
 
+# Events we wait on before auto-approving a version.
+WEBHOOK_EVENTS_BLOCKING_AUTO_APPROVAL = [
+    *WEBHOOK_EVENTS_AGGREGATED_BY_ON_VERSION_SCANNED,
+    WEBHOOK_ON_VERSION_SCANNED,
+]
+
 # Number of retries before we give up on the scanners for a version.
 WEBHOOK_MAX_RETRIES = 12
 

src/olympia/lib/settings_base.py

@@ -832,6 +832,7 @@ def get_language_url_map():
     'olympia.files.tasks.repack_fileupload': {'queue': 'devhub'},
     'olympia.scanners.tasks.call_webhooks_during_validation': {'queue': 'devhub'},
     'olympia.scanners.tasks.call_webhooks_on_listing_changed': {'queue': 'devhub'},
+    'olympia.scanners.tasks.call_webhooks_on_version_scanned': {'queue': 'devhub'},
     'olympia.scanners.tasks.run_actions_for_scanner_result': {'queue': 'devhub'},
     'olympia.scanners.tasks.run_narc_on_version': {'queue': 'devhub'},
     'olympia.scanners.tasks.run_yara': {'queue': 'devhub'},

src/olympia/reviewers/tests/test_models.py

@@ -26,7 +26,11 @@
     version_factory,
 )
 from olympia.blocklist.models import BlockVersion
-from olympia.constants.scanners import WEBHOOK, WEBHOOK_DURING_VALIDATION
+from olympia.constants.scanners import (
+    WEBHOOK,
+    WEBHOOK_DURING_VALIDATION,
+    WEBHOOK_ON_VERSION_SCANNED,
+)
 from olympia.files.models import File, FileValidation, WebextPermission
 from olympia.promoted.models import (
     PromotedAddon,
@@ -1505,6 +1509,22 @@ def test_check_is_waiting_on_scanners_result_without_matched_rules(self):
         )
         assert AutoApprovalSummary.check_is_waiting_on_scanners(self.version) is True
 
+    def test_check_is_waiting_on_scanners_on_version_scanned_event(self):
+        webhook = ScannerWebhook.objects.create(name='some-scanner')
+        webhook.update(modified=self.days_ago(1))
+        event = ScannerWebhookEvent.objects.create(
+            event=WEBHOOK_ON_VERSION_SCANNED, webhook=webhook
+        )
+        assert AutoApprovalSummary.check_is_waiting_on_scanners(self.version) is True
+
+        ScannerResult.objects.create(
+            version=self.version,
+            scanner=WEBHOOK,
+            webhook_event=event,
+            results={'matchedRules': []},
+        )
+        assert AutoApprovalSummary.check_is_waiting_on_scanners(self.version) is False
+
     def test_check_is_waiting_on_scanners_multiple_events_all_complete(self):
         webhook = ScannerWebhook.objects.create(name='some-scanner')
         webhook.update(modified=self.days_ago(1))

src/olympia/scanners/management/commands/retry_versions_waiting_on_scanners.py

@@ -4,19 +4,19 @@
 from django.core.management.base import BaseCommand
 
 import olympia.core.logger
-from olympia.constants.scanners import (
-    WEBHOOK_EVENTS_BLOCKING_AUTO_APPROVAL,
-    WEBHOOK_MAX_RETRIES,
+from olympia.constants.scanners import WEBHOOK_MAX_RETRIES
+from olympia.scanners.tasks import (
+    scanner_webhook_events_to_wait_for,
+    wait_for_scanner_results,
 )
-from olympia.scanners.tasks import wait_for_scanner_results
 from olympia.versions.models import Version
 
 
 log = olympia.core.logger.getLogger('z.scanners.retry_versions_waiting_on_scanners')
 
 # How long `wait_for_scanner_results` can keep retrying a version: the initial
-# countdown plus one delay per retry.
-MIN_AGE = timedelta(
+# countdown plus one delay per retry, for each of the two steps we wait in.
+MIN_AGE = 2 * timedelta(
     seconds=settings.SCANNER_WEBHOOK_RETRY_DELAY * (WEBHOOK_MAX_RETRIES + 1)
 )
 
@@ -64,7 +64,7 @@ def handle(self, *args, **options):
 
             wait_for_scanner_results.delay(
                 version_pk=version.pk,
-                event_ids=WEBHOOK_EVENTS_BLOCKING_AUTO_APPROVAL,
+                event_ids=scanner_webhook_events_to_wait_for(version),
             )
             log.info(
                 'Scheduled wait_for_scanner_results again for version %s.', version.pk

src/olympia/scanners/migrations/0093_alter_scannerwebhookevent_event.py

@@ -0,0 +1,24 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('scanners', '0092_alter_scannerwebhookevent_event'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='scannerwebhookevent',
+            name='event',
+            field=models.PositiveSmallIntegerField(
+                choices=[
+                    (1, 'during_validation'),
+                    (2, 'on_source_code_uploaded'),
+                    (3, 'on_version_created'),
+                    (4, 'push'),
+                    (5, 'on_listing_changed'),
+                    (6, 'on_version_scanned'),
+                ]
+            ),
+        ),
+    ]