Handle ConnectionResetError from rabbitmq.

As seen in
https://ubuntu-archive-team.ubuntu.com/proposed-migration/log/noble/2024-04-01/12:52:27.log

Having an entire britney run bail because of a connection reset is a bad
outcome!

Instead, catch this exception and avoid adding the test in question to the
list of queued tests (we can pick it up on the next run).

Possibly we should do more clever handling of a ConnectionResetError such as
reconnecting, but this is a minimum fix that will stop britney from aborting.
less-recipients
Steve Langasek 9 months ago
parent 07cc085a77
commit 13f2ba4277

@ -1118,9 +1118,11 @@ class AutopkgtestPolicy(BasePolicy):
If huge is true, then the request will be put into the -huge instead of
normal queue.
Return True if successfully queued, False if not.
'''
if self.options.dry_run or self.dry_run:
return
return True
params = {'triggers': triggers}
if self.options.adt_ppas:
@ -1135,9 +1137,12 @@ class AutopkgtestPolicy(BasePolicy):
if self.amqp_channel:
import amqplib.client_0_8 as amqp
params = json.dumps(params)
self.amqp_channel.basic_publish(amqp.Message(src + '\n' + params,
delivery_mode=2), # persistent
routing_key=qname)
try:
self.amqp_channel.basic_publish(amqp.Message(src + '\n' + params,
delivery_mode=2), # persistent
routing_key=qname)
except ConnectionResetError:
return False
else:
# for file-based submission, triggers are space separated
params['triggers'] = [' '.join(params['triggers'])]
@ -1145,6 +1150,7 @@ class AutopkgtestPolicy(BasePolicy):
assert self.amqp_file
with open(self.amqp_file, 'a') as f:
f.write('%s:%s %s\n' % (qname, src, params))
return True
def pkg_test_request(self, src, arch, full_triggers, huge=False):
'''Request one package test for one particular trigger
@ -1214,11 +1220,12 @@ class AutopkgtestPolicy(BasePolicy):
self.logger.info('Test %s/%s for %s is already pending, not queueing', src, arch, trigger)
else:
self.logger.info('Requesting %s autopkgtest on %s to verify %s', src, arch, trigger)
arch_list.append(arch)
arch_list.sort()
self.send_test_request(src, arch, full_triggers, huge=huge)
# save pending.json right away, so that we don't re-request if britney crashes
self.save_pending_json()
if self.send_test_request(src, arch, full_triggers, huge=huge):
# save pending.json right away, so that we don't re-request
# if britney crashes
arch_list.append(arch)
arch_list.sort()
self.save_pending_json()
def result_in_baseline(self, src, arch):
'''Get the result for src on arch in the baseline

Loading…
Cancel
Save