merge trunk.

debian/changelog

@@ -1,9 +1,15 @@
 ubuntu-dev-tools (0.129) UNRELEASED; urgency=low
 
-  * Remove get-branches, grab-attachments, lp-project-upload, lp-list-bugs,
+  [ Colin Watson ]
-    lp-set-dup and lp-shell which are now included in lptools.
+  * syncpackage: Convert to new LP API, with --no-lp available for the old
+    style of operation.
+  * syncpackage: Require -f/--force option to overwrite Ubuntu changes.
 
- -- Jelmer Vernooij <jelmer@ubuntu.com>  Wed, 17 Aug 2011 12:47:59 +0200
+  [ Jelmer Vernooij ]
+  * Remove get-branches, grab-attachments, lp-project-upload, lp-list-
+    bugs, lp-set-dup and lp-shell which are now included in lptools.
+
+ -- Jelmer Vernooij <jelmer@debian.org>  Thu, 01 Sep 2011 19:38:53 +0200
 
 ubuntu-dev-tools (0.128) unstable; urgency=low
 

doc/requestsync.1

@@ -32,6 +32,11 @@ This check is only performed if \fBrequestsync\fR is allowed to use the LP API
 answer the question about upload permissions honestly to determine if a team
 with approval rights is to be subscribed to the bug.
 
+If you have permission to upload the package directly, then you may prefer
+to use \fBsyncpackage\fR instead to copy the package using the Launchpad
+API. At some future point, \fBrequestsync\fR will be changed to do this
+automatically.
+
 .PP
 \fBrequestsync\fR uses launchpadlib authentication to file its requests.
 

doc/syncpackage.1

@@ -1,12 +1,12 @@
 .TH SYNCPACKAGE "1" "June 2010" "ubuntu-dev-tools"
 .SH NAME
-syncpackage \- helper to prepare .changes file to upload synced packages
+syncpackage \- copy source packages from Debian to Ubuntu
 .SH SYNOPSIS
 .B syncpackage
 [\fIoptions\fR] \fI<.dsc URL/path or package name>\fR
 .SH DESCRIPTION
-\fBsyncpackage\fR generates a changes file to be directly uploaded to Ubuntu
+\fBsyncpackage\fR causes a source package to be copied from Debian to
-primary archive or PPA starting from a pristine Debian package.
+Ubuntu.
 .PP
 \fBsyncpackage\fR allows you to upload files with the same checksums of the
 Debian ones, as the common script used by Ubuntu archive administrators does,
@@ -15,14 +15,15 @@ this way you can preserve source files integrity between the two distributions.
 \fBsyncpackage\fR will detect source tarballs with mismatching checksums
 and will automatically create fake syncs instead.
 .SH WARNING
-The use of \fBsyncpackage\fR is discouraged by the Ubuntu Archive
+The use of \fBsyncpackage \-\-no\-lp\fR, which generates a changes file to
-Administrators, as it introduces an unnecessary window for error.
+be directly uploaded to the Ubuntu primary archive or a PPA, is discouraged
-In future launchpad will offer the ability to directly perform sync
+by the Ubuntu Archive Administrators, as it introduces an unnecessary window
-requests, without a developer having to do an upload.
+for error.
-Until then, it's advised to use \fBsyncpackage\fR with caution, and
+This only exists for backward compatibility, for unusual corner cases, and
-request syncs via bugs with
+for uploads to archives other than the Ubuntu primary archive.
-.BR requestsync (1)
+Omitting this option will cause Launchpad to perform the sync request
-where possible.
+directly, which is the preferred method for uploads to the Ubuntu primary
+archive.
 .SH OPTIONS
 .TP
 \fB\-h\fR, \fB\-\-help\fR
@@ -43,13 +44,22 @@ Specify the component to sync from.
 \fB\-v\fR, \fB\-\-verbose\fR
 Display more progress information.
 .TP
+.B \-\-no\-lp
+Construct sync locally rather than letting Launchpad copy the package
+directly (not recommended).
+.TP
+\fB\-l\fI INSTANCE\fR, \fB\-\-lpinstance\fR=\fIINSTANCE\fR
+Launchpad instance to connect to (default: production).
+.TP
 \fB\-n\fI UPLOADER_NAME\fR, \fB\-\-uploader\-name\fR=\fIUPLOADER_NAME\fR
 Use UPLOADER_NAME as the name of the maintainer for this upload instead
 of evaluating DEBFULLNAME and UBUMAIL.
+This option may only be used in \fB\-\-no\-lp\fR mode.
 .TP
 \fB\-e\fI UPLOADER_EMAIL\fR, \fB\-\-uploader\-email\fR=\fIUPLOADER_EMAIL\fR
 Use UPLOADER_EMAIL as the email address of the maintainer for this
 upload instead of evaluating DEBEMAIL and UBUMAIL.
+This option may only be used in \fB\-\-no\-lp\fR mode.
 .TP
 \fB\-k\fI KEYID\fR, \fB\-\-key\fR=\fIKEYID\fR
 Specify the key ID to be used for signing.
@@ -60,6 +70,12 @@ Do not sign the upload.
 \fB\-b\fI BUG\fR, \fB\-\-bug\fR=\fIBUG\fR
 Mark a Launchpad bug as being fixed by this upload.
 .TP
+\fB\-f\fR, \fB\-\-force\fR
+Force sync over the top of Ubuntu changes.
+If a sync is blacklisted because of a source tarball mismatch,
+\fB\-\-force\fR can be used to override the blacklist, when doing a
+local sync (\fB\-\-no\-lp\fR).
+.TP
 .B \-d \fIDEBIAN_MIRROR\fR, \fB\-\-debian\-mirror\fR=\fIDEBIAN_MIRROR\fR
 Use the specified mirror.
 Should be in the form \fBhttp://ftp.debian.org/debian\fR.
@@ -75,6 +91,9 @@ back to the default mirror.
 .B \-\-no\-conf
 Do not read any configuration files, or configuration from environment
 variables.
+.TP
+.B \-\-simulate
+Show what would be done, but don't actually do it.
 .SH ENVIRONMENT
 .TP
 .BR DEBFULLNAME ", " DEBEMAIL ", " UBUMAIL

requestsync

@@ -239,6 +239,10 @@ def main():
     if not sponsorship:
         sponsorship = needSponsorship(srcpkg, ubuntu_component, release)
 
+    if not sponsorship and not ffe:
+        print >> sys.stderr, ('Consider using syncpackage(1) for syncs that '
+                              'do not require feature freeze exceptions.')
+
     # Check for existing package reports
     if not newsource:
         checkExistingReports(srcpkg)

syncpackage

@@ -20,14 +20,17 @@
 #
 # ##################################################################
 
-import debian.deb822
+import codecs
-import debian.debian_support
 import optparse
 import os
+import re
 import shutil
 import sys
+import urllib
 
+import debian.debian_support
 from devscripts.logger import Logger
+from lazr.restfulclient.errors import HTTPError
 
 from ubuntutools.archive import (DebianSourcePackage, UbuntuSourcePackage,
                                  DownloadError)
@@ -36,7 +39,9 @@ from ubuntutools.requestsync.mail import (getDebianSrcPkg
                                           as requestsync_mail_getDebianSrcPkg)
 from ubuntutools.requestsync.lp import getDebianSrcPkg, getUbuntuSrcPkg
 from ubuntutools.lp import udtexceptions
-from ubuntutools.lp.lpapicache import Launchpad
+from ubuntutools.lp.lpapicache import Distribution, Launchpad
+from ubuntutools.misc import split_release_pocket
+from ubuntutools.question import YesNoQuestion
 from ubuntutools import subprocess
 
 
@@ -110,7 +115,7 @@ def add_fixed_bugs(changes, bugs):
     return "\n".join(changes + [""])
 
 def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror,
-             keyid=None):
+             keyid=None, simulate=False, force=False):
     uploader = name + " <" + email + ">"
 
     src_pkg.pull_dsc()
@@ -136,9 +141,15 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror,
 
     cur_ver = ubuntu_ver.get_related_debian_version()
     if ubuntu_ver.is_modified_in_ubuntu():
+        if not force:
+            Logger.error('--force is required to discard Ubuntu changes.')
+            sys.exit(1)
+
         Logger.warn('Overwriting modified Ubuntu version %s, '
                     'setting current version to %s',
                     ubuntu_ver.full_version, cur_ver.full_version)
+    if simulate:
+        return
 
     try:
         src_pkg.pull()
@@ -248,8 +259,13 @@ def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror):
     """Download the specified source package.
     dist, version, component, mirror can all be None.
     """
+    if mirror is None:
+        mirrors = []
+    else:
+        mirrors = [mirror]
+
     if package.endswith('.dsc'):
-        return DebianSourcePackage(dscfile=package, mirrors=[mirror])
+        return DebianSourcePackage(dscfile=package, mirrors=mirrors)
 
     if dist is None:
         dist = "unstable"
@@ -290,7 +306,118 @@ def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror):
     assert component in ('main', 'contrib', 'non-free')
 
     return DebianSourcePackage(package, version.full_version, component,
-                               mirrors=[mirror])
+                               mirrors=mirrors)
+
+def copy(src_pkg, release, simulate=False, force=False):
+    """Copy a source package from Debian to Ubuntu using the Launchpad API."""
+    ubuntu = Distribution('ubuntu')
+    debian_archive = Distribution('debian').getArchive()
+    ubuntu_archive = ubuntu.getArchive()
+    if release is None:
+        ubuntu_series = ubuntu.getDevelopmentSeries().name
+        ubuntu_pocket = 'Release'
+    else:
+        ubuntu_series, ubuntu_pocket = split_release_pocket(release)
+
+    # Ensure that the provided Debian version actually exists.
+    try:
+        debian_archive.getPublishedSources(
+            source_name=src_pkg.source,
+            version=src_pkg.version.full_version,
+            exact_match=True)[0]
+    except IndexError:
+        Logger.error('Debian version %s does not exist!', src_pkg.version)
+        sys.exit(1)
+
+    try:
+        ubuntu_spph = getUbuntuSrcPkg(src_pkg.source,
+                                      ubuntu_series, ubuntu_pocket)
+        ubuntu_pkg = UbuntuSourcePackage(src_pkg.source,
+                                         ubuntu_spph.getVersion(),
+                                         ubuntu_spph.getComponent(),
+                                         mirrors=[])
+
+        Logger.normal('Source %s -> %s/%s: current version %s, new version %s',
+                      src_pkg.source, ubuntu_series, ubuntu_pocket,
+                      ubuntu_pkg.version, src_pkg.version)
+
+        ubuntu_version = Version(ubuntu_pkg.version.full_version)
+        if not force and ubuntu_version.is_modified_in_ubuntu():
+            Logger.error('--force is required to discard Ubuntu changes.')
+            sys.exit(1)
+
+        # Check whether a fakesync would be required.
+        src_pkg.pull_dsc()
+        ubuntu_pkg.pull_dsc()
+        if not src_pkg.dsc.compare_dsc(ubuntu_pkg.dsc):
+            Logger.error('The checksums of the Debian and Ubuntu packages '
+                         'mismatch. A fake sync using --no-lp is required.')
+            sys.exit(1)
+    except udtexceptions.PackageNotFoundException:
+        Logger.normal('Source %s -> %s/%s: not in Ubuntu, new version %s',
+                      src_pkg.source, ubuntu_series, ubuntu_pocket,
+                      src_pkg.version)
+    if simulate:
+        return
+
+    answer = YesNoQuestion().ask("Sync this package", "no")
+    if answer != "yes":
+        return
+
+    try:
+        ubuntu_archive.copyPackage(
+            source_name=src_pkg.source,
+            version=src_pkg.version.full_version,
+            from_archive=debian_archive,
+            to_series=ubuntu_series,
+            to_pocket=ubuntu_pocket,
+            include_binaries=False)
+    except HTTPError, error:
+        Logger.error("HTTP Error %s: %s", error.response.status,
+                     error.response.reason)
+        Logger.error(error.content)
+        sys.exit(1)
+
+    Logger.normal('Request succeeded; you should get an e-mail once it is '
+                  'processed.')
+
+def is_blacklisted(query):
+    """"Determine if package "query" is in the sync blacklist
+    Returns True or a string of all relevant comments if blacklisted,
+            False if not
+    """
+    # LP:
+    # TODO: Refactor when LP: #833080 is fixed
+    series = Launchpad.distributions['ubuntu'].current_series
+    diffs = series.getDifferencesTo(source_package_name_filter=query,
+                                    status='Needs attention')
+    if len(diffs) == 0:
+        comments = series.getDifferenceComments(source_package_name=query)
+        comment = '; '.join(c.body_text for c in comments)
+        if comment:
+            return comment
+        return True
+
+    # Old blacklist:
+    url = 'http://people.canonical.com/~ubuntu-archive/sync-blacklist.txt'
+    with codecs.EncodedFile(urllib.urlopen(url), 'UTF-8') as f:
+        applicable_comments = []
+        for line in f:
+            if not line.strip():
+                applicable_comments = []
+                continue
+            m = re.match(r'^\s*([a-z0-9.+-]+)?\s*(?:#\s*(.+)?)?$', line)
+            source, comment = m.groups()
+            if source and query == source:
+                if comment:
+                    applicable_comments.append(comment)
+                if applicable_comments:
+                    return u'; '.join(applicable_comments)
+                else:
+                    return True
+            elif comment:
+                applicable_comments.append(comment)
+    return False
 
 def main():
     usage = "%prog [options] <.dsc URL/path or package name>"
@@ -312,6 +439,15 @@ def main():
     parser.add_option("-v", "--verbose",
                       dest="verbose", action="store_true", default=False,
                       help="Display more progress information.")
+    parser.add_option("--no-lp",
+                      dest="lp", action="store_false", default=True,
+                      help="Construct sync locally rather than letting "
+                           "Launchpad copy the package directly (not "
+                           "recommended).")
+    parser.add_option('-l', '--lpinstance', metavar='INSTANCE',
+                      dest='lpinstance', default=None,
+                      help='Launchpad instance to connect to '
+                           '(default: production).')
     parser.add_option("-n", "--uploader-name",
                       dest="uploader_name", default=None,
                       help="Use UPLOADER_NAME as the name of the maintainer "
@@ -330,6 +466,10 @@ def main():
                       dest="bugs", action="append", default=list(),
                       help="Mark Launchpad bug BUG as being fixed by this "
                            "upload.")
+    parser.add_option("-f", "--force",
+                      dest="force", action="store_true", default=False,
+                      help="Force sync over the top of Ubuntu changes "
+                           "or a fake-sync when blacklisted.")
     parser.add_option('-D', '--debian-mirror', metavar='DEBIAN_MIRROR',
                       dest='debian_mirror',
                       help='Preferred Debian mirror '
@@ -343,6 +483,10 @@ def main():
     parser.add_option('--no-conf',
                       dest='no_conf', default=False, action='store_true',
                       help="Don't read config files or environment variables.")
+    parser.add_option('--simulate',
+                      dest='simulate', default=False, action='store_true',
+                      help="Show what would be done, but don't actually do "
+                           "it.")
 
     (options, args) = parser.parse_args()
 
@@ -362,6 +506,14 @@ def main():
                      'It should be one of main, contrib, or non-free.'
                      % options.component)
 
+    if options.lp and options.uploader_name:
+        parser.error('Uploader name can only be overridden using --no-lp.')
+    if options.lp and options.uploader_email:
+        parser.error('Uploader email address can only be overridden using '
+                     '--no-lp.')
+    # --key, --dont-sign, --debian-mirror, and --ubuntu-mirror are just
+    # ignored with options.lp, and do not require warnings.
+
     Logger.verbose = options.verbose
     config = UDTConfig(options.no_conf)
     if options.debian_mirror is None:
@@ -373,19 +525,47 @@ def main():
     if options.uploader_email is None:
         options.uploader_email = ubu_email(export=False)[1]
 
-    Launchpad.login_anonymously()
+    if options.lp:
-    if options.release is None:
+        if args[0].endswith('.dsc'):
-        options.release = Launchpad.distributions["ubuntu"].current_series.name
+            parser.error('.dsc files can only be synced using --no-lp.')
 
-    os.environ['DEB_VENDOR'] = 'Ubuntu'
+    if options.lpinstance is None:
+        options.lpinstance = config.get_value('LPINSTANCE')
+    try:
+        Launchpad.login(service=options.lpinstance, api_version='devel')
+    except IOError:
+        sys.exit(1)
+
+    if options.release is None:
+        ubuntu = Launchpad.distributions["ubuntu"]
+        options.release = ubuntu.current_series.name
 
     src_pkg = fetch_source_pkg(args[0], options.dist, options.debversion,
                                options.component, options.release,
                                options.debian_mirror)
 
-    sync_dsc(src_pkg, options.dist, options.release, options.uploader_name,
+    blacklisted = is_blacklisted(src_pkg.source)
-             options.uploader_email, options.bugs, options.ubuntu_mirror,
+    if blacklisted:
-             options.keyid)
+        Logger.error("Source package is blacklisted")
+        if isinstance(blacklisted, basestring):
+            Logger.error(u"Reason: %s", blacklisted)
+        Logger.error("If you think this package shouldn't be blacklisted, "
+                     "please file a bug explaining your reasoning and "
+                     "subscribe ~ubuntu-archive.")
+        if options.force and not options.lp:
+            Logger.warn(u"Forcing fake-sync, overriding blacklist.")
+        else:
+            Logger.error("--force and --no-lp are required to override the "
+                         "blacklist, if this package needs a fakesync.")
+            sys.exit(1)
+
+    if options.lp:
+        copy(src_pkg, options.release, options.simulate, options.force)
+    else:
+        os.environ['DEB_VENDOR'] = 'Ubuntu'
+        sync_dsc(src_pkg, options.dist, options.release, options.uploader_name,
+                 options.uploader_email, options.bugs, options.ubuntu_mirror,
+                 options.keyid, options.simulate, options.force)
 
 if __name__ == "__main__":
     main()

ubuntutools/archive.py

@@ -95,6 +95,30 @@ class Dsc(debian.deb822.Dsc):
             return hash_func.hexdigest() == digest
         return False
 
+    def compare_dsc(self, other):
+        """Check whether any files in these two dscs that have the same name
+        also have the same checksum."""
+        for field, key in (('Checksums-Sha256', 'sha256'),
+                           ('Checksums-Sha1', 'sha1'),
+                           ('Files', 'md5sum')):
+            if field not in self or field not in other:
+                continue
+            our_checksums = \
+                dict((entry['name'], (int(entry['size']), entry[key]))
+                     for entry in self[field])
+            their_checksums = \
+                dict((entry['name'], (int(entry['size']), entry[key]))
+                     for entry in other[field])
+            for name, (size, checksum) in our_checksums.iteritems():
+                if name not in their_checksums:
+                    # file only in one dsc
+                    continue
+                if (size != their_checksums[name][0] or
+                    checksum != their_checksums[name][1]):
+                    return False
+            return True # one checksum is good enough
+        return True
+
 
 class SourcePackage(object):
     """Base class for source package downloading.

ubuntutools/config.py

@@ -16,7 +16,6 @@
 # PERFORMANCE OF THIS SOFTWARE.
 
 import os
-import os.path
 import pwd
 import re
 import shlex

ubuntutools/lp/lpapicache.py

@@ -53,7 +53,7 @@ _POCKETS = ('Release', 'Security', 'Updates', 'Proposed', 'Backports')
 class _Launchpad(object):
     '''Singleton for LP API access.'''
 
-    def login(self, service=service):
+    def login(self, service=service, api_version=api_version):
         '''Enforce a non-anonymous login.'''
         if not self.logged_in:
             try:
@@ -201,7 +201,7 @@ class Distribution(BaseWrapper):
             cached = Distribution(Launchpad.distributions[dist])
         return cached
 
-    def getArchive(self, archive = None):
+    def getArchive(self, archive=None):
         '''
         Returns an Archive object for the requested archive.
         Raises a ArchiveNotFoundException if the archive doesn't exist.
@@ -278,7 +278,7 @@ class Archive(BaseWrapper):
         if '_srcpkgs' not in self.__dict__:
             self._srcpkgs = dict()
 
-    def getSourcePackage(self, name, series = None, pocket = 'Release'):
+    def getSourcePackage(self, name, series=None, pocket='Release'):
         '''
         Returns a SourcePackagePublishingHistory object for the most
         recent source package in the distribution 'dist', series and
@@ -332,6 +332,25 @@ class Archive(BaseWrapper):
 
         return self._srcpkgs[(name, series.name, pocket)]
 
+    def copyPackage(self, source_name, version, from_archive, to_pocket,
+                    to_series=None, include_binaries=False):
+        '''Copy a single named source into this archive.
+
+        Asynchronously copy a specific version of a named source to the
+        destination archive if necessary.  Calls to this method will return
+        immediately if the copy passes basic security checks and the copy
+        will happen sometime later with full checking.
+        '''
+
+        self._lpobject.copyPackage(
+            source_name=source_name,
+            version=version,
+            from_archive=from_archive._lpobject,
+            to_pocket=to_pocket,
+            to_series=to_series,
+            include_binaries=include_binaries
+            )
+
 
 class SourcePackagePublishingHistory(BaseWrapper):
     '''
@@ -427,7 +446,7 @@ class MetaPersonTeam(MetaWrapper):
             except HTTPError, error:
                 if error.response.status == 401:
                     # Anonymous login
-                    cls._me  = None
+                    cls._me = None
                 else:
                     raise
         return cls._me

ubuntutools/misc.py

@@ -25,7 +25,6 @@
 # Modules.
 import locale
 import os
-import os.path
 import sys
 
 import distro_info

ubuntutools/requestsync/lp.py

@@ -34,11 +34,11 @@ def getDebianSrcPkg(name, release):
 
     return debian_archive.getSourcePackage(name, release)
 
-def getUbuntuSrcPkg(name, release):
+def getUbuntuSrcPkg(name, release, pocket='Release'):
     ubuntu = Distribution('ubuntu')
     ubuntu_archive = ubuntu.getArchive()
 
-    return ubuntu_archive.getSourcePackage(name, release)
+    return ubuntu_archive.getSourcePackage(name, release, pocket)
 
 def needSponsorship(name, component, release):
     '''

ubuntutools/test/test_config.py

@@ -16,7 +16,6 @@
 
 import __builtin__
 import os
-import os.path
 import sys
 from StringIO import StringIO