autopkgtest: Add "huge" AMQP queues

Add new "huge" AMQP queues

Whenever a package like glibc or perl gets uploaded, these trigger thousands of
tests that fill the queues for several days. This can't (solely) be addressed
by adding more capacity due to the sheer size. The main annoyance of this is
that this blocks propagation of every other package in -proposed during that
time.

AMQP has no concept of priorities within a queue (they are strictly FIFO), so
for packages that trigger a huge number of tests, send these to the new
debci-$release-huge-$arch queues instead.

LP: #1647948

policies/autopkgtest.py

@@ -175,8 +175,10 @@ class AutopkgtestPolicy(BasePolicy):
         pkg_arch_result = {}
         for arch in self.adt_arches:
             # request tests (unless they were already requested earlier or have a result)
-            for (testsrc, testver) in self.tests_for_source(source_name, source_data_srcdist.version, arch):
-                self.pkg_test_request(testsrc, arch, trigger)
+            tests = self.tests_for_source(source_name, source_data_srcdist.version, arch)
+            is_huge = len(tests) > 20
+            for (testsrc, testver) in tests:
+                self.pkg_test_request(testsrc, arch, trigger, huge=is_huge)
                 (result, real_ver, url) = self.pkg_test_result(testsrc, testver, arch, trigger)
                 pkg_arch_result.setdefault((testsrc, real_ver), {})[arch] = (result, url)
 
@@ -574,9 +576,12 @@ class AutopkgtestPolicy(BasePolicy):
                 result[1] = ver
                 result[2] = stamp
 
-    def send_test_request(self, src, arch, trigger):
-        '''Send out AMQP request for testing src/arch for trigger'''
+    def send_test_request(self, src, arch, trigger, huge=False):
+        '''Send out AMQP request for testing src/arch for trigger
 
+        If huge is true, then the request will be put into the -huge instead of
+        normal queue.
+        '''
         if self.options.dry_run:
             return
 
@@ -584,6 +589,8 @@ class AutopkgtestPolicy(BasePolicy):
         if self.options.adt_ppas:
             params['ppas'] = self.options.adt_ppas
             qname = 'debci-ppa-%s-%s' % (self.options.series, arch)
+        elif huge:
+            qname = 'debci-huge-%s-%s' % (self.options.series, arch)
         else:
             qname = 'debci-%s-%s' % (self.options.series, arch)
         params = json.dumps(params)
@@ -595,11 +602,12 @@ class AutopkgtestPolicy(BasePolicy):
             with open(self.amqp_file, 'a') as f:
                 f.write('%s:%s %s\n' % (qname, src, params))
 
-    def pkg_test_request(self, src, arch, trigger):
+    def pkg_test_request(self, src, arch, trigger, huge=False):
         '''Request one package test for one particular trigger
 
         trigger is "pkgname/version" of the package that triggers the testing
-        of src.
+        of src. If huge is true, then the request will be put into the -huge
+        instead of normal queue.
 
         This will only be done if that test wasn't already requested in a
         previous run (i. e. not already in self.pending_tests) or there already
@@ -634,7 +642,7 @@ class AutopkgtestPolicy(BasePolicy):
                              (src, arch, trigger))
             arch_list.append(arch)
             arch_list.sort()
-            self.send_test_request(src, arch, trigger)
+            self.send_test_request(src, arch, trigger, huge=huge)
 
     def check_ever_passed(self, src, arch):
         '''Check if tests for src ever passed on arch'''

tests/test_autopkgtest.py

@@ -1388,6 +1388,27 @@ class T(TestBase):
             }
         )
 
+    def test_huge_number_of_tests(self):
+        '''package triggers huge number of tests'''
+
+        for i in range(30):
+            self.data.add('green%i' % i, False, {'Depends': 'libgreen1'}, testsuite='autopkgtest')
+
+        self.do_test(
+            [('libgreen1', {'Version': '2', 'Source': 'green'}, 'autopkgtest')],
+            {'green': (True, {'green': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'},
+                              'green0': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'},
+                              'green29': {'amd64': 'RUNNING-ALWAYSFAIL', 'i386': 'RUNNING-ALWAYSFAIL'},
+                             })
+            },
+            )
+
+        # requests should all go into the -huge queues
+        self.assertEqual([x for x in self.amqp_requests if 'huge' not in x], [])
+        for i in range(30):
+            for arch in ['i386', 'amd64']:
+                self.assertIn('debci-huge-series-%s:green%i {"triggers": ["green/2"]}' %
+                              (arch, i), self.amqp_requests)
 
     ################################################################
     # Tests for hint processing