Merge from trunk

ack-sync

@@ -51,7 +51,7 @@ def strip_epoch(version):
     version = ':'.join(parts)
     return version
 
-def LogCall(command):
+def log_call(command):
     command = map(str, command)
     logging.info("Running %s", " ".join(command))
     return command
@@ -88,22 +88,22 @@ def get_source(package, version, section, dist, uploader_name, uploader_email,
         sys.exit(1)
     return dsc_file
 
-def build_source(dist, dsc_file):
+def build_source(dist, dsc_file, pbuilder, sbuild):
     try:
         if sbuild:
             cmd = ["sbuild", "-d", dist, "-A", dsc_file]
-            subprocess.check_call(LogCall(cmd))
+            subprocess.check_call(log_call(cmd))
         else:
             if not os.path.isdir("buildresult"):
                 os.makedirs("buildresult")
             cmd = ["sudo", "-E", "DIST=" + dist, pbuilder, "--build",
                    "--buildresult", "buildresult", dsc_file]
-            subprocess.check_call(LogCall(cmd))
+            subprocess.check_call(log_call(cmd))
     except subprocess.CalledProcessError:
         print >> sys.stderr, "E: %s failed to build." % (dsc_file)
         sys.exit(1)
 
-def test_install(dist, dsc_file):
+def test_install(dist, dsc_file, sbuild, lvm):
     changes_files = glob.glob(os.path.splitext(dsc_file)[0]+"_*.changes")
     changes_file = ""
 
@@ -122,9 +122,9 @@ def test_install(dist, dsc_file):
                "restricted multiverse", changes_file]
         if sbuild:
             lvm_volume = lvm + "/" + dist + "_chroot"
-            subprocess.check_call(LogCall(cmd + ["--lvm-volume="+lvm_volume]))
+            subprocess.check_call(log_call(cmd + ["--lvm-volume="+lvm_volume]))
         else:
-            subprocess.check_call(LogCall(cmd + ["--pbuilder"]))
+            subprocess.check_call(log_call(cmd + ["--pbuilder"]))
     except subprocess.CalledProcessError:
         print >> sys.stderr, "E: %s failed to install. Please check log" % \
                              (changes_file)
@@ -154,8 +154,8 @@ def unsubscribe_sponsors(launchpad, bug):
 
 
 def ack_sync(bug_numbers, all_package, all_version, all_section, update,
-             all_uploader_email, key, upload, lpinstance, verbose=False,
+             all_uploader_email, key, upload, lpinstance, pbuilder, sbuild, lvm,
-             silent=False):
+             piuparts, verbose=False, silent=False):
     launchpad = get_launchpad("ubuntu-dev-tools", server=lpinstance)
     # TODO: use release-info (once available)
     series = launchpad.distributions["ubuntu"].current_series
@@ -164,10 +164,10 @@ def ack_sync(bug_numbers, all_package, all_version, all_section, update,
     # update pbuilder
     if update:
         if sbuild:
-            subprocess.call(LogCall(["sbuild-update", dist]))
+            subprocess.call(log_call(["sbuild-update", dist]))
         else:
             cmd = ["sudo", "-E", "DIST=" + dist, pbuilder, "--update"]
-            subprocess.call(LogCall(cmd))
+            subprocess.call(log_call(cmd))
 
     for bug_number in bug_numbers:
         bug = launchpad.bugs[bug_number]
@@ -270,10 +270,10 @@ def ack_sync(bug_numbers, all_package, all_version, all_section, update,
         env['DEB_VENDOR'] = 'Ubuntu'
         subprocess.check_call(["dpkg-source", "-x", dsc_file], env=env)
 
-        build_source(dist, dsc_file)
+        build_source(dist, dsc_file, pbuilder, sbuild)
 
         if piuparts:
-            test_install(dist, dsc_file)
+            test_install(dist, dsc_file, sbuild, lvm)
 
         print bug.title
         print '%s (was %s)' % (task.status, old_status)
@@ -430,7 +430,8 @@ def main():
     #TODO: Support WORKDIR
 
     ack_sync(bug_numbers, package, version, section, update, uploader_email,
-             key, upload, lpinstance, verbose, silent)
+             key, upload, lpinstance, pbuilder, sbuild, lvm, piuparts, verbose,
+             silent)
 
 if __name__ == '__main__':
     main()

check-mir

@@ -0,0 +1,143 @@
+#!/usr/bin/python
+#
+# Check components of build dependencies and warn about universe/multiverse
+# ones, for a package destined for main/restricted
+#
+# Copyright (C) 2011 Canonical
+#
+# Authors:
+#  Martin Pitt
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; version 3.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+import apt
+import sys
+import os.path
+
+def check_support(apt_cache, pkgname, alt=False):
+    '''Check if pkgname is in main or restricted.
+
+    This prints messages if a package is not in main/restricted, or only
+    partially (i. e. source in main, but binary in universe).
+    '''
+    if alt:
+        prefix = '  ... alternative ' + pkgname
+    else:
+        prefix = ' * ' + pkgname
+
+    try:
+        pkg = apt_cache[pkgname]
+    except KeyError:
+        print >> sys.stderr, prefix, 'does not exist (pure virtual?)'
+        return False
+    
+    section = pkg.candidate.section
+    if section.startswith('universe') or section.startswith('multiverse'):
+        # check if the source package is in main and thus will only need binary
+        # promotion
+        source_records = apt.apt_pkg.SourceRecords()
+        if not source_records.lookup(pkg.candidate.source_name):
+            print >> sys.stderr, 'ERROR: Cannot lookup source package for', \
+                                 pkg.name
+            print prefix, 'package is in', section.split('/')[0]
+            return False
+        src = apt.apt_pkg.TagSection(source_records.record)
+        if (src['Section'].startswith('universe') or
+            src['Section'].startswith('multiverse')):
+            print prefix, 'binary and source package is in', \
+                  section.split('/')[0]
+            return False
+        else:
+            print prefix, 'is in', section.split('/')[0] + ', but its source', \
+                  pkg.candidate.source_name, \
+                  ('is already in main; file an ubuntu-archive bug for '
+                   'promoting the current preferred alternative')
+            return True
+
+    if alt:
+        print prefix, 'is already in main; consider preferring it'
+
+    return True
+
+def check_build_dependencies(apt_cache, control):
+    print 'Checking support status of build dependencies...'
+
+    any_unsupported = False
+
+    for field in ('Build-Depends', 'Build-Depends-Indep'):
+        if field not in control.section:
+            continue
+        for or_group in apt.apt_pkg.parse_src_depends(control.section[field]):
+            pkgname = or_group[0][0]
+            if not check_support(apt_cache, pkgname):
+                # check non-preferred alternatives
+                for altpkg in or_group[1:]:
+                    if check_support(apt_cache, altpkg[0], alt=True):
+                        break
+                else:
+                    any_unsupported = True
+
+    return any_unsupported
+
+def check_binary_dependencies(apt_cache, control):
+    any_unsupported = False
+
+    print '\nChecking support status of binary dependencies...'
+    while True:
+        try:
+            control.next()
+        except StopIteration:
+            break
+
+        for field in ('Depends', 'Pre-Depends', 'Recommends'):
+            if field not in control.section:
+                continue
+            for or_group in apt.apt_pkg.parse_depends(control.section[field]):
+                pkgname = or_group[0][0]
+                if pkgname.startswith('$'):
+                    continue
+                if not check_support(apt_cache, pkgname):
+                    # check non-preferred alternatives
+                    for altpkg in or_group[1:]:
+                        if check_support(apt_cache, altpkg[0], alt=True):
+                            break
+                    else:
+                        any_unsupported = True
+
+    return any_unsupported
+
+def main():
+    apt_cache = apt.Cache()
+
+    if not os.path.exists('debian/control'):
+        print >> sys.stderr, ('debian/control not found. You need to run '
+                              'this tool in a source package directory')
+        sys.exit(1)
+
+    # get build dependencies from debian/control
+    control = apt.apt_pkg.TagFile(open('debian/control'))
+    control.next()
+
+    unsupported_build_deps = check_build_dependencies(apt_cache, control)
+    unsupported_binary_deps = check_binary_dependencies(apt_cache, control)
+
+    if unsupported_build_deps or unsupported_binary_deps:
+        print ('\nPlease check https://wiki.ubuntu.com/MainInclusionProcess if '
+               'this source package needs to get into in main/restricted, or '
+               'reconsider if the package really needs above dependencies.')
+    else:
+        print 'All dependencies are supported in main or restricted.'
+
+if __name__ == '__main__':
+    main()

debian/changelog

@@ -1,4 +1,27 @@
-ubuntu-dev-tools (0.109) UNRELEASED; urgency=low
+ubuntu-dev-tools (0.112) UNRELEASED; urgency=low
+
+  * New source package downloading framework in ubuntutools.archive. Use in
+    many scripts.
+  * pull-lp-source: str() exceptions before passing to Logger (LP: #695523)
+
+ -- Stefano Rivera <stefanor@ubuntu.com>  Sat, 15 Jan 2011 13:32:12 +0200
+
+ubuntu-dev-tools (0.111) natty; urgency=low
+
+  * ubuntutools/test/test_help.py: Blacklist --help test for check-mir, it
+    does not have help. Fixes FTBFS on the buildd.
+
+ -- Martin Pitt <martin.pitt@ubuntu.com>  Thu, 13 Jan 2011 20:15:41 -0600
+
+ubuntu-dev-tools (0.110) natty; urgency=low
+
+  * doc/check-mir.1: Fix typo.
+  * check-mir: Check binary dependencies, too.
+  * debian/control: Add check-mir to package description.
+
+ -- Martin Pitt <martin.pitt@ubuntu.com>  Thu, 13 Jan 2011 19:28:20 -0600
+
+ubuntu-dev-tools (0.109) natty; urgency=low
 
   [ Stefano Rivera ]
   * Convert debian/copyright to DEP5, make sure all scripts are listed
@@ -13,8 +36,6 @@ ubuntu-dev-tools (0.109) UNRELEASED; urgency=low
     - Added ubuntu-dev-tools.5
     - Support this in many u-d-t scripts, and update manpages.
     - Deprecate old configuration environment variables.
-  * New source package downloading framework in ubuntutools.archive. Use in
-    many scripts.
   * Support the combined "Name <email>" format in UBUMAIL, DEBFULLNAME, and
     DEBEMAIL. (LP: #665202)
   * Add the beginnings of a test suite. (LP: #690386)
@@ -29,10 +50,10 @@ ubuntu-dev-tools (0.109) UNRELEASED; urgency=low
   * edit-patch: Don't let cat error through if debian/source/format doesn't
     exist.
   * pull-debian-debdiff: Rewrite in Python, and use snapshot.debian.org.
-  * pull-lp-source:
+  * pull-lp-source: Support -d (LP: #681699)
-    - Support -d (LP: #681699)
-    - str() exceptions before passing to Logger (LP: #695523)
   * suspicious-source: Whitelist Python source code.
+  * import-bug-from-debian: Add --package option, for importing bugs from
+    pseudo-packages.
 
   [ Michael Bienia ]
   * ubuntutools/lp/lpapicache.py: Allow easier selection of 'staging' as LP
@@ -56,7 +77,12 @@ ubuntu-dev-tools (0.109) UNRELEASED; urgency=low
   * add "add-patch" that provides the non-interactive version of
     edit-patch
 
- -- Stefano Rivera <stefanor@ubuntu.com>  Thu, 30 Dec 2010 17:14:09 +0200
+  [ Martin Pitt ]
+  * Add check-mir script: Check components of build dependencies and warn
+    about universe/multiverse ones, for a package destined for
+    main/restricted. Add doc/check-mir.1 manpage.
+
+ -- Martin Pitt <martin.pitt@ubuntu.com>  Thu, 13 Jan 2011 19:16:33 -0600
 
 ubuntu-dev-tools (0.108) experimental; urgency=low
 

debian/control

@@ -2,8 +2,7 @@ Source: ubuntu-dev-tools
 Section: devel
 Priority: optional
 Maintainer: Ubuntu Developers <ubuntu-dev-team@lists.alioth.debian.org>
-Uploaders: Luca Falavigna <dktrkranz@debian.org>,
+Uploaders: Benjamin Drung <bdrung@debian.org>
-           Benjamin Drung <bdrung@ubuntu.com>
 Vcs-Bzr: lp:ubuntu-dev-tools
 Vcs-Browser: https://code.launchpad.net/~ubuntu-dev/ubuntu-dev-tools/trunk
 Build-Depends: dctrl-tools,
@@ -23,7 +22,6 @@ Build-Depends: dctrl-tools,
                python-setuptools,
                python-soappy,
                python-unittest2
-DM-Upload-Allowed: yes
 X-Python-Version: >= 2.5
 Homepage: https://launchpad.net/ubuntu-dev-tools
 Standards-Version: 3.9.1
@@ -68,6 +66,7 @@ Description: useful tools for Ubuntu developers
  .
   - 404main - used to check what components a package's deps are in, for
     doing a main inclusion report for example.
+  - check-mir - check support status of build/binary dependencies
   - check-symbols - will compare and give you a diff of the exported symbols of
     all .so files in a binary package.
   - dch-repeat - used to repeat a change log into an older release.

doc/check-mir.1

@@ -0,0 +1,18 @@
+.TH check\-mir "1" "13 January 2011" "ubuntu-dev-tools"
+.SH NAME
+check\-mir \- check support status of dependencies
+
+.SH SYNOPSIS
+.B check\-mir
+
+.SH DESCRIPTION
+This script checks if any of a package's build or binary dependencies is
+in universe/multiverse. If the source package is destined for Ubuntu main or
+restricted, these either need to be eliminated or need to be promoted to main,
+following \fBhttps://wiki.ubuntu.com/MainInclusionProcess\fR.
+
+There are no options, just run it in a source package directory.
+
+.SH AUTHOR
+.B check\-mir
+was written by Martin Pitt <martin.pitt@ubuntu.com>.

doc/import-bug-from-debian.1

@@ -4,7 +4,7 @@ import\-bug\-from\-debian \- Import bugs from Debian's BTS, and file
 them against Ubuntu in LP.
 
 .SH SYNOPSIS
-.B import\-bug\-from\-debian \fR[\fB\-nb\fR] \fIbug\fR...
+.B import\-bug\-from\-debian \fR[\fIoptions\fR] \fIbug\fR...
 .br
 .B import\-bug\-from\-debian \-h
 
@@ -28,6 +28,11 @@ Display a help message and exit.
 Use the specified instance of Launchpad (e.g. "staging"), instead of
 the default of "production".
 .TP
+.B \-p \fIPACKAGE\fR, \fB\-\-package\fR=\fIPACKAGE\fR
+Launchpad package to file bug against, if not the same source package
+name as Debian.
+Useful for importing removal bugs filed against \fBftp.debian.org\fR.
+.TP
 .B \-\-no\-conf
 Do not read any configuration files, or configuration from environment
 variables.

import-bug-from-debian

@@ -54,6 +54,10 @@ def main():
     parser.add_option("-n", "--dry-run",
                       help=SUPPRESS_HELP,
                       dest="lpinstance", action="store_const", const="staging")
+    parser.add_option("-p", "--package", metavar="PACKAGE",
+                      help="Launchpad package to file bug against "
+                           "(default: Same as Debian)",
+                      dest="package", default=None)
     parser.add_option("--no-conf", dest="no_conf", default=False,
                       help="Don't read config files or environment variables.",
                       action="store_true")
@@ -96,15 +100,22 @@ def main():
     for bug in bugs:
         bug = bug.value
         package = bug.package
+        ubupackage = package
+        if options.package:
+            ubupackage = options.package
         bug_num = bug.bug_num
         subject = bug.subject
         log = debbugs.get_bug_log(bug_num)
         summary = log[0][0]
-        target = ubuntu.getSourcePackage(name=package)
+        target = ubuntu.getSourcePackage(name=ubupackage)
         u_bug = launchpad.bugs.createBug(target=target, title=subject,
-                description="Imported from Debian bug %d:\n\n%s"
+                description='Imported from Debian bug '
-                    % (bug_num, summary))
+                            'http://bugs.debian.org/%d:\n\n%s'
-        d_task = u_bug.addTask(target=debian.getSourcePackage(name=package))
+                            % (bug_num, summary))
+        d_sp = debian.getSourcePackage(name=package)
+        if d_sp is None and options.package:
+            d_sp = debian.getSourcePackage(name=options.package)
+        d_task = u_bug.addTask(target=d_sp)
         d_watch = u_bug.addWatch(remote_bug=bug_num, bug_tracker=lp_debbugs)
         d_task.bug_watch = d_watch
         d_task.lp_save()

setup.py

@@ -15,6 +15,7 @@ if os.path.exists(changelog):
 
 scripts = ['404main',
            'backportpackage',
+           'check-mir',
            'check-symbols',
            'dch-repeat',
            'dgetlp',

ubuntutools/sponsor_patch/sponsor_patch.py

@@ -218,6 +218,39 @@ def apply_patch(task, patch):
                 edit = True
     return edit
 
+def get_open_ubuntu_bug_task(launchpad, bug):
+    """Returns an open Ubuntu bug task for a given Launchpad bug.
+
+    The bug task needs to be open (not complete) and target Ubuntu. The user
+    will be ask to select one if multiple open Ubuntu bug task exits for the
+    bug.
+    """
+    bug_tasks = [BugTask(x, launchpad) for x in bug.bug_tasks]
+    ubuntu_tasks = [x for x in bug_tasks if x.is_ubuntu_task()]
+    if len(ubuntu_tasks) == 0:
+        Logger.error("No Ubuntu bug task found on bug #%i." % (bug.id))
+        sys.exit(1)
+    elif len(ubuntu_tasks) == 1:
+        task = ubuntu_tasks[0]
+    if len(ubuntu_tasks) > 1:
+        task_list = [t.get_short_info() for t in ubuntu_tasks]
+        Logger.info("%i Ubuntu tasks exist for bug #%i.\n%s", len(ubuntu_tasks),
+                    bug.id, "\n".join(task_list))
+        open_ubuntu_tasks = [x for x in ubuntu_tasks if not x.is_complete()]
+        if len(open_ubuntu_tasks) == 1:
+            task = open_ubuntu_tasks[0]
+        else:
+            Logger.normal("https://launchpad.net/bugs/%i has %i Ubuntu tasks:" \
+                          % (bug.id, len(ubuntu_tasks)))
+            for i in xrange(len(ubuntu_tasks)):
+                print "%i) %s" % (i + 1,
+                                  ubuntu_tasks[i].get_package_and_series())
+            selected = input_number("To which Ubuntu tasks do the patch belong",
+                                    1, len(ubuntu_tasks))
+            task = ubuntu_tasks[selected - 1]
+    Logger.info("Selected Ubuntu task: %s" % (task.get_short_info()))
+    return task
+
 def sponsor_patch(bug_number, build, builder, edit, keyid, lpinstance, update,
                   upload, workdir, verbose=False):
     workdir = os.path.expanduser(workdir)
@@ -239,33 +272,7 @@ def sponsor_patch(bug_number, build, builder, edit, keyid, lpinstance, update,
     #pylint: enable=E1101
 
     (patch, branch) = get_patch_or_branch(bug)
-
+    task = get_open_ubuntu_bug_task(launchpad, bug)
-    bug_tasks = [BugTask(x, launchpad) for x in bug.bug_tasks]
-    ubuntu_tasks = [x for x in bug_tasks if x.is_ubuntu_task()]
-    if len(ubuntu_tasks) == 0:
-        Logger.error("No Ubuntu bug task found on bug #%i." % (bug_number))
-        sys.exit(1)
-    elif len(ubuntu_tasks) == 1:
-        task = ubuntu_tasks[0]
-    if len(ubuntu_tasks) > 1:
-        if verbose:
-            Logger.info("%i Ubuntu tasks exist for bug #%i." % \
-                        (len(ubuntu_tasks), bug_number))
-            for task in ubuntu_tasks:
-                print task.get_short_info()
-        open_ubuntu_tasks = [x for x in ubuntu_tasks if x.is_complete()]
-        if len(open_ubuntu_tasks) == 1:
-            task = open_ubuntu_tasks[0]
-        else:
-            Logger.normal("https://launchpad.net/bugs/%i has %i Ubuntu tasks:" \
-                          % (bug_number, len(ubuntu_tasks)))
-            for i in xrange(len(ubuntu_tasks)):
-                print "%i) %s" % (i + 1,
-                                  ubuntu_tasks[i].get_package_and_series())
-            selected = input_number("To which Ubuntu tasks do the patch belong",
-                                    1, len(ubuntu_tasks))
-            task = ubuntu_tasks[selected - 1]
-    Logger.info("Selected Ubuntu task: %s" % (task.get_short_info()))
 
     dsc_file = task.download_source()
     assert os.path.isfile(dsc_file), "%s does not exist." % (dsc_file)

ubuntutools/test/test_help.py

@@ -25,6 +25,7 @@ import setup
 from ubuntutools.test import unittest
 
 BLACKLIST = {
+    'check-mir': 'No Help',
     'check-symbols': 'No Help',
     'edit-patch': 'No Help',
     'get-build-deps': 'No Help, runs sudo',