From 13f2ba4277ac78ec78c06b860fad7a6860867e2c Mon Sep 17 00:00:00 2001
From: Steve Langasek <steve.langasek@canonical.com>
Date: Mon, 1 Apr 2024 18:14:35 -0700
Subject: [PATCH] Handle ConnectionResetError from rabbitmq.

As seen in
https://ubuntu-archive-team.ubuntu.com/proposed-migration/log/noble/2024-04-01/12:52:27.log

Having an entire britney run bail because of a connection reset is a bad
outcome!

Instead, catch this exception and avoid adding the test in question to the
list of queued tests (we can pick it up on the next run).

Possibly we should do more clever handling of a ConnectionResetError such as
reconnecting, but this is a minimum fix that will stop britney from aborting.
---
 britney2/policies/autopkgtest.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/britney2/policies/autopkgtest.py b/britney2/policies/autopkgtest.py
index 568a816..94a2afb 100644
--- a/britney2/policies/autopkgtest.py
+++ b/britney2/policies/autopkgtest.py
@@ -1118,9 +1118,11 @@ class AutopkgtestPolicy(BasePolicy):
 
         If huge is true, then the request will be put into the -huge instead of
         normal queue.
+
+        Return True if successfully queued, False if not.
         '''
         if self.options.dry_run or self.dry_run:
-            return
+            return True
 
         params = {'triggers': triggers}
         if self.options.adt_ppas:
@@ -1135,9 +1137,12 @@ class AutopkgtestPolicy(BasePolicy):
         if self.amqp_channel:
             import amqplib.client_0_8 as amqp
             params = json.dumps(params)
-            self.amqp_channel.basic_publish(amqp.Message(src + '\n' + params,
-                                                         delivery_mode=2),  # persistent
-                                            routing_key=qname)
+            try:
+                self.amqp_channel.basic_publish(amqp.Message(src + '\n' + params,
+                                                             delivery_mode=2),  # persistent
+                                                routing_key=qname)
+            except ConnectionResetError:
+                return False
         else:
             # for file-based submission, triggers are space separated
             params['triggers'] = [' '.join(params['triggers'])]
@@ -1145,6 +1150,7 @@ class AutopkgtestPolicy(BasePolicy):
             assert self.amqp_file
             with open(self.amqp_file, 'a') as f:
                 f.write('%s:%s %s\n' % (qname, src, params))
+        return True
 
     def pkg_test_request(self, src, arch, full_triggers, huge=False):
         '''Request one package test for one particular trigger
@@ -1214,11 +1220,12 @@ class AutopkgtestPolicy(BasePolicy):
             self.logger.info('Test %s/%s for %s is already pending, not queueing', src, arch, trigger)
         else:
             self.logger.info('Requesting %s autopkgtest on %s to verify %s', src, arch, trigger)
-            arch_list.append(arch)
-            arch_list.sort()
-            self.send_test_request(src, arch, full_triggers, huge=huge)
-            # save pending.json right away, so that we don't re-request if britney crashes
-            self.save_pending_json()
+            if self.send_test_request(src, arch, full_triggers, huge=huge):
+                # save pending.json right away, so that we don't re-request
+                # if britney crashes
+                arch_list.append(arch)
+                arch_list.sort()
+                self.save_pending_json()
 
     def result_in_baseline(self, src, arch):
         '''Get the result for src on arch in the baseline