Handle ConnectionResetError from rabbitmq.

As seen in
https://ubuntu-archive-team.ubuntu.com/proposed-migration/log/noble/2024-04-01/12:52:27.log

Having an entire britney run bail because of a connection reset is a bad
outcome!

Instead, catch this exception and avoid adding the test in question to the
list of queued tests (we can pick it up on the next run).

Possibly we should do more clever handling of a ConnectionResetError such as
reconnecting, but this is a minimum fix that will stop britney from aborting.
less-recipients
Steve Langasek 10 months ago
parent 07cc085a77
commit 13f2ba4277

@ -1118,9 +1118,11 @@ class AutopkgtestPolicy(BasePolicy):
If huge is true, then the request will be put into the -huge instead of If huge is true, then the request will be put into the -huge instead of
normal queue. normal queue.
Return True if successfully queued, False if not.
''' '''
if self.options.dry_run or self.dry_run: if self.options.dry_run or self.dry_run:
return return True
params = {'triggers': triggers} params = {'triggers': triggers}
if self.options.adt_ppas: if self.options.adt_ppas:
@ -1135,9 +1137,12 @@ class AutopkgtestPolicy(BasePolicy):
if self.amqp_channel: if self.amqp_channel:
import amqplib.client_0_8 as amqp import amqplib.client_0_8 as amqp
params = json.dumps(params) params = json.dumps(params)
self.amqp_channel.basic_publish(amqp.Message(src + '\n' + params, try:
delivery_mode=2), # persistent self.amqp_channel.basic_publish(amqp.Message(src + '\n' + params,
routing_key=qname) delivery_mode=2), # persistent
routing_key=qname)
except ConnectionResetError:
return False
else: else:
# for file-based submission, triggers are space separated # for file-based submission, triggers are space separated
params['triggers'] = [' '.join(params['triggers'])] params['triggers'] = [' '.join(params['triggers'])]
@ -1145,6 +1150,7 @@ class AutopkgtestPolicy(BasePolicy):
assert self.amqp_file assert self.amqp_file
with open(self.amqp_file, 'a') as f: with open(self.amqp_file, 'a') as f:
f.write('%s:%s %s\n' % (qname, src, params)) f.write('%s:%s %s\n' % (qname, src, params))
return True
def pkg_test_request(self, src, arch, full_triggers, huge=False): def pkg_test_request(self, src, arch, full_triggers, huge=False):
'''Request one package test for one particular trigger '''Request one package test for one particular trigger
@ -1214,11 +1220,12 @@ class AutopkgtestPolicy(BasePolicy):
self.logger.info('Test %s/%s for %s is already pending, not queueing', src, arch, trigger) self.logger.info('Test %s/%s for %s is already pending, not queueing', src, arch, trigger)
else: else:
self.logger.info('Requesting %s autopkgtest on %s to verify %s', src, arch, trigger) self.logger.info('Requesting %s autopkgtest on %s to verify %s', src, arch, trigger)
arch_list.append(arch) if self.send_test_request(src, arch, full_triggers, huge=huge):
arch_list.sort() # save pending.json right away, so that we don't re-request
self.send_test_request(src, arch, full_triggers, huge=huge) # if britney crashes
# save pending.json right away, so that we don't re-request if britney crashes arch_list.append(arch)
self.save_pending_json() arch_list.sort()
self.save_pending_json()
def result_in_baseline(self, src, arch): def result_in_baseline(self, src, arch):
'''Get the result for src on arch in the baseline '''Get the result for src on arch in the baseline

Loading…
Cancel
Save