Update pylint and flake8 unittests

Import improvements from https://github.com/bdrung/snippets

ubuntutools/test/__init__.py

@@ -1,6 +1,5 @@
-# Test suite for ubuntutools
-#
 # Copyright (C) 2010, Stefano Rivera <stefanor@ubuntu.com>
+# Copyright (C) 2017, Benjamin Drung <benjamin.drung@profitbricks.com>
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
@@ -14,6 +13,9 @@
 # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 # PERFORMANCE OF THIS SOFTWARE.
 
+"""Test suite for ubuntutools"""
+
+import inspect
 import os
 import sys
 
@@ -53,3 +55,16 @@ def get_source_files():
         else:
             files.append(code_file)
     return files
+
+
+def unittest_verbosity():
+    """Return the verbosity setting of the currently running unittest
+    program, or None if none is running.
+    """
+    frame = inspect.currentframe()
+    while frame:
+        self = frame.f_locals.get("self")
+        if isinstance(self, unittest.TestProgram):
+            return self.verbosity
+        frame = frame.f_back
+    return None  # pragma: no cover

ubuntutools/test/pylint.conf

@@ -1,3 +1,33 @@
+[MASTER]
+
+# Pickle collected data for later comparisons.
+persistent=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
+confidence=HIGH
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once).You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use"--disable=all --enable=classes
+# --disable=W"
+disable=locally-disabled
+
+
+[REPORTS]
+
+# Tells whether to display a full report or only the messages
+reports=no
+
+
 [TYPECHECK]
 
 # List of classes names for which member attributes should not be checked
@@ -5,6 +35,7 @@
 # lpapicache classes, urlparse
 ignored-classes=Launchpad,BaseWrapper,PersonTeam,Distribution,Consumer,Credentials,ParseResult,apt_pkg,apt_pkg.Dependency,apt_pkg.BaseDependency
 
+
 [FORMAT]
 
 # Maximum number of characters on a single line.
@@ -14,6 +45,7 @@ max-line-length=99
 # tab).
 indent-string='    '
 
+
 [BASIC]
 
 # Allow variables called e, f, lp

ubuntutools/test/test_flake8.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2017, Benjamin Drung <bdrung@debian.org>
+# Copyright (C) 2017-2018, Benjamin Drung <bdrung@debian.org>
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
@@ -17,19 +17,34 @@
 import subprocess
 import sys
 
-from ubuntutools.test import get_source_files, unittest
+from ubuntutools.test import get_source_files, unittest, unittest_verbosity
 
 
 class Flake8TestCase(unittest.TestCase):
+    """
+    This unittest class provides a test that runs the flake8 code
+    checker (which combines pycodestyle and pyflakes) on the Python
+    source code. The list of source files is provided by the
+    get_source_files() function.
+    """
+
     def test_flake8(self):
-        "Test: Run flake8 on Python source code"
-        with open('/proc/self/cmdline', 'r') as cmdline_file:
-            python_binary = cmdline_file.read().split('\0')[0]
-        cmd = [python_binary, '-m', 'flake8', '--max-line-length=99'] + get_source_files()
-        sys.stderr.write("Running following command:\n{}\n".format(" ".join(cmd)))
+        """Test: Run flake8 on Python source code"""
+        cmd = [sys.executable, "-m", "flake8", "--max-line-length=99"] + get_source_files()
+        if unittest_verbosity() >= 2:
+            sys.stderr.write("Running following command:\n{}\n".format(" ".join(cmd)))
         process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE, close_fds=True)
 
         out, err = process.communicate()
-        self.assertFalse(err, "Unexpected standard error from flake8 run:\n" + err.decode())
-        self.assertFalse(out, "flake8 found issues:\n" + out.decode())
+        if process.returncode != 0:  # pragma: no cover
+            msgs = []
+            if err:
+                msgs.append("flake8 exited with code {} and has unexpected output on stderr:\n{}"
+                            .format(process.returncode, err.decode().rstrip()))
+            if out:
+                msgs.append("flake8 found issues:\n{}".format(out.decode().rstrip()))
+            if not msgs:
+                msgs.append("flake8 exited with code {} and has no output on stdout or stderr."
+                            .format(process.returncode))
+            self.fail("\n".join(msgs))

ubuntutools/test/test_pylint.py

@@ -1,7 +1,5 @@
-# test_pylint.py - Run pylint in errors-only mode.
-#
 # Copyright (C) 2010, Stefano Rivera <stefanor@ubuntu.com>
-# Copyright (C) 2017, Benjamin Drung <bdrung@ubuntu.com>
+# Copyright (C) 2017-2018, Benjamin Drung <bdrung@ubuntu.com>
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
@@ -15,26 +13,56 @@
 # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 # PERFORMANCE OF THIS SOFTWARE.
 
+"""test_pylint.py - Run pylint in errors-only mode."""
+
+import os
+import re
 import sys
 
-from ubuntutools.test import get_source_files, unittest
+from ubuntutools.test import get_source_files, unittest, unittest_verbosity
 from ubuntutools import subprocess
 
+CONFIG = os.path.join(os.path.dirname(__file__), "pylint.conf")
+
 
 class PylintTestCase(unittest.TestCase):
+    """
+    This unittest class provides a test that runs the pylint code check
+    on the Python source code. The list of source files is provided by
+    the get_source_files() function and pylint is configured via a
+    config file.
+    """
+
     def test_pylint(self):
-        "Test: Run pylint on Python source code"
-        if sys.version_info[0] == 3:
-            pylint_binary = 'pylint3'
-        else:
-            pylint_binary = 'pylint'
-        cmd = [pylint_binary, '--rcfile=ubuntutools/test/pylint.conf', '-E',
-               '--reports=n', '--confidence=HIGH', '--'] + get_source_files()
-        sys.stderr.write("Running following command:\n{}\n".format(" ".join(cmd)))
-        try:
-            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-        except subprocess.CalledProcessError as e:
-            self.fail(
-                '%s crashed (%d).  Error output:\n%s' %
-                (pylint_binary, e.returncode, e.output.decode())
-            )
+        """Test: Run pylint on Python source code"""
+
+        cmd = [sys.executable, "-m", "pylint", "--rcfile=" + CONFIG,
+               "-E", "--"] + get_source_files()
+        if unittest_verbosity() >= 2:
+            sys.stderr.write("Running following command:\n{}\n".format(" ".join(cmd)))
+        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                                   close_fds=True)
+        out, err = process.communicate()
+
+        if process.returncode != 0:  # pragma: no cover
+            # Strip trailing summary (introduced in pylint 1.7). This summary might look like:
+            #
+            # ------------------------------------
+            # Your code has been rated at 10.00/10
+            #
+            out = re.sub("^(-+|Your code has been rated at .*)$", "", out.decode(),
+                         flags=re.MULTILINE).rstrip()
+
+            # Strip logging of used config file (introduced in pylint 1.8)
+            err = re.sub("^Using config file .*\n", "", err.decode()).rstrip()
+
+            msgs = []
+            if err:
+                msgs.append("pylint exited with code {} and has unexpected output on stderr:\n{}"
+                            .format(process.returncode, err))
+            if out:
+                msgs.append("pylint found issues:\n{}".format(out))
+            if not msgs:
+                msgs.append("pylint exited with code {} and has no output on stdout or stderr."
+                            .format(process.returncode))
+            self.fail("\n".join(msgs))