From 775274ca8940bc8daa3c536feacceec24f6f49c7 Mon Sep 17 00:00:00 2001
From: Martin Pitt <martin.pitt@canonical.com>
Date: Thu, 17 Sep 2015 12:31:13 +0200
Subject: [PATCH] Autopkgtest: Ignore results without package/version

We often get "tmpfail" results (repeated failure to start cloud instance, etc.)
with no package/version at all. Stop attributing them to the latest pending
request for that package, as that has already messed up some results. With
moving to tracking test triggers in testinfo.jar and running multiple test
requests for each triggering kernel version it becomes completely impossible to
interpret anything into a tmpfail result without testpkg-version, so just
ignore them.

This will leave some orphaned entries in pending.txt and thus require manual
retries after fixing the tmpfail reason. But this needs to happen anyway, so
this does not complicate operation but instead shows those as "in progress"
instead of "regression".
---
 autopkgtest.py            | 14 +++++---------
 tests/test_autopkgtest.py | 10 +++++-----
 2 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/autopkgtest.py b/autopkgtest.py
index 3c4f49b..a708e31 100644
--- a/autopkgtest.py
+++ b/autopkgtest.py
@@ -386,12 +386,11 @@ class AutoPackageTest(object):
                 srcver = tar.extractfile('testpkg-version').read().decode().strip()
                 (ressrc, ver) = srcver.split()
         except (KeyError, ValueError, tarfile.TarError) as e:
-            self.log_error('%s is damaged: %s' % (url, str(e)))
-            # we can't just ignore this, as it would leave an orphaned request
-            # in pending.txt; consider it tmpfail
-            exitcode = 16
-            ressrc = src
-            ver = None
+            self.log_error('%s is damaged, ignoring: %s' % (url, str(e)))
+            # ignore this; this will leave an orphaned request in pending.txt
+            # and thus require manual retries after fixing the tmpfail, but we
+            # can't just blindly attribute it to some pending test.
+            return
 
         if src != ressrc:
             self.log_error('%s is a result for package %s, but expected package %s' %
@@ -408,9 +407,6 @@ class AutoPackageTest(object):
         # remove matching test requests, remember triggers
         satisfied_triggers = set()
         for pending_ver, pending_archinfo in self.pending_tests.get(src, {}).copy().items():
-            # if we encounter a tmpfail above, attribute it to the pending test
-            if ver is None:
-                ver = pending_ver
             # don't consider newer requested versions
             if apt_pkg.version_compare(pending_ver, ver) <= 0:
                 try:
diff --git a/tests/test_autopkgtest.py b/tests/test_autopkgtest.py
index e6ee505..406a6d1 100755
--- a/tests/test_autopkgtest.py
+++ b/tests/test_autopkgtest.py
@@ -985,9 +985,9 @@ lightgreen 1 i386 green 3
         self.assertEqual(self.pending_requests, '')
 
     def test_tmpfail(self):
-        '''tmpfail result is considered a failure'''
+        '''tmpfail results'''
 
-        # one tmpfail result without testpkg-version
+        # one tmpfail result without testpkg-version, should be ignored
         self.swift.set_results({'autopkgtest-series': {
             'series/i386/l/lightgreen/20150101_100000@': (0, 'lightgreen 1'),
             'series/i386/l/lightgreen/20150101_100101@': (16, None),
@@ -997,8 +997,8 @@ lightgreen 1 i386 green 3
 
         self.do_test(
             [('lightgreen', {'Version': '2', 'Depends': 'libgreen1 (>= 1)'}, 'autopkgtest')],
-            {'lightgreen': (False, {'lightgreen 2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}})})
-        self.assertEqual(self.pending_requests, '')
+            {'lightgreen': (False, {'lightgreen 2': {'amd64': 'REGRESSION', 'i386': 'RUNNING'}})})
+        self.assertEqual(self.pending_requests, 'lightgreen 2 i386 lightgreen 2\n')
 
         # one more tmpfail result, should not confuse britney with None version
         self.swift.set_results({'autopkgtest-series': {
@@ -1006,7 +1006,7 @@ lightgreen 1 i386 green 3
         }})
         self.do_test(
             [],
-            {'lightgreen': (False, {'lightgreen 2': {'amd64': 'REGRESSION', 'i386': 'REGRESSION'}})})
+            {'lightgreen': (False, {'lightgreen 2': {'amd64': 'REGRESSION', 'i386': 'RUNNING'}})})
         with open(os.path.join(self.data.path, 'data/series-proposed/autopkgtest/results.cache')) as f:
             contents = f.read()
         self.assertNotIn('null', contents)