From b5fd32ec7b6cc5f52868c224403221e79aee7ccc Mon Sep 17 00:00:00 2001
From: Martin Pitt <martin.pitt@canonical.com>
Date: Thu, 4 Feb 2016 15:13:17 +0100
Subject: [PATCH] Fail hard on unexpected HTTP errors

In the CI train we sometimes run into transient "HTTP Error 502: Proxy Error".
As we don't keep a results.cache there, this leads to retrying tests for which
we already have a result in swift, but can't download it. Treat this as a hard
failure now, to let the next britney run try again. This will also tell us if
we need to handle any other status code than 200, 204 (empty result), or 401
(container does not exist).
---
 autopkgtest.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/autopkgtest.py b/autopkgtest.py
index e1d2f1b..825c491 100644
--- a/autopkgtest.py
+++ b/autopkgtest.py
@@ -22,6 +22,7 @@ import json
 import tarfile
 import io
 import re
+import sys
 import urllib.parse
 from urllib.request import urlopen
 
@@ -340,14 +341,23 @@ class AutoPackageTest(object):
             elif f.getcode() == 204:  # No content
                 result_paths = []
             else:
-                self.log_error('Failure to fetch swift results from %s: %u' %
-                               (url, f.getcode()))
-                f.close()
-                return
+                # we should not ever end up here as we expect a HTTPError in
+                # other cases; e. g. 3XX is something that tells us to adjust
+                # our URLS, so fail hard on those
+                raise NotImplementedError('fetch_swift_results(%s): cannot handle HTTP code %i' %
+                                          (url, f.getcode()))
             f.close()
         except IOError as e:
-            self.log_error('Failure to fetch swift results from %s: %s' % (url, str(e)))
-            return
+            # 401 "Unauthorized" is swift's way of saying "container does not exist"
+            if hasattr(e, 'code') and e.code == 401:
+                self.log_verbose('fetch_swift_results: %s does not exist yet or is inaccessible' % url)
+                return
+            # Other status codes are usually a transient
+            # network/infrastructure failure. Ignoring this can lead to
+            # re-requesting tests which we already have results for, so
+            # fail hard on this and let the next run retry.
+            self.log_error('FATAL: Failure to fetch swift results from %s: %s' % (url, str(e)))
+            sys.exit(1)
 
         for p in result_paths:
             self.fetch_one_result(
@@ -366,11 +376,11 @@ class AutoPackageTest(object):
                 tar_bytes = io.BytesIO(f.read())
                 f.close()
             else:
-                self.log_error('Failure to fetch %s: %u' % (url, f.getcode()))
-                return
+                raise NotImplementedError('fetch_one_result(%s): cannot handle HTTP code %i' %
+                                          (url, f.getcode()))
         except IOError as e:
-            self.log_error('Failure to fetch %s: %s' % (url, str(e)))
-            return
+            self.log_error('FATAL: Failure to fetch %s: %s' % (url, str(e)))
+            sys.exit(1)
 
         try:
             with tarfile.open(None, 'r', tar_bytes) as tar: