* syncpackage: Convert to new LP API, with --no-lp available for the old

style of operation.
* syncpackage: Require -f/--force option to overwrite Ubuntu changes.

debian/changelog

@@ -1,3 +1,11 @@
+ubuntu-dev-tools (0.129) UNRELEASED; urgency=low
+
+  * syncpackage: Convert to new LP API, with --no-lp available for the old
+    style of operation.
+  * syncpackage: Require -f/--force option to overwrite Ubuntu changes.
+
+ -- Colin Watson <cjwatson@ubuntu.com>  Tue, 16 Aug 2011 16:40:22 +0100
+
 ubuntu-dev-tools (0.128) unstable; urgency=low
 
   [ Stefano Rivera ]

doc/requestsync.1

@@ -32,6 +32,11 @@ This check is only performed if \fBrequestsync\fR is allowed to use the LP API
 answer the question about upload permissions honestly to determine if a team
 with approval rights is to be subscribed to the bug.
 
+If you have permission to upload the package directly, then you may prefer
+to use \fBsyncpackage\fR instead to copy the package using the Launchpad
+API. At some future point, \fBrequestsync\fR will be changed to do this
+automatically.
+
 .PP
 \fBrequestsync\fR uses launchpadlib authentication to file its requests.
 

doc/syncpackage.1

@@ -1,12 +1,12 @@
 .TH SYNCPACKAGE "1" "June 2010" "ubuntu-dev-tools"
 .SH NAME
-syncpackage \- helper to prepare .changes file to upload synced packages
+syncpackage \- copy source packages from Debian to Ubuntu
 .SH SYNOPSIS
 .B syncpackage
 [\fIoptions\fR] \fI<.dsc URL/path or package name>\fR
 .SH DESCRIPTION
-\fBsyncpackage\fR generates a changes file to be directly uploaded to Ubuntu
+\fBsyncpackage\fR causes a source package to be copied from Debian to
-primary archive or PPA starting from a pristine Debian package.
+Ubuntu.
 .PP
 \fBsyncpackage\fR allows you to upload files with the same checksums of the
 Debian ones, as the common script used by Ubuntu archive administrators does,
@@ -15,14 +15,15 @@ this way you can preserve source files integrity between the two distributions.
 \fBsyncpackage\fR will detect source tarballs with mismatching checksums
 and will automatically create fake syncs instead.
 .SH WARNING
-The use of \fBsyncpackage\fR is discouraged by the Ubuntu Archive
+The use of \fBsyncpackage \-\-no\-lp\fR, which generates a changes file to
-Administrators, as it introduces an unnecessary window for error.
+be directly uploaded to the Ubuntu primary archive or a PPA, is discouraged
-In future launchpad will offer the ability to directly perform sync
+by the Ubuntu Archive Administrators, as it introduces an unnecessary window
-requests, without a developer having to do an upload.
+for error.
-Until then, it's advised to use \fBsyncpackage\fR with caution, and
+This only exists for backward compatibility, for unusual corner cases, and
-request syncs via bugs with
+for uploads to archives other than the Ubuntu primary archive.
-.BR requestsync (1)
+Omitting this option will cause Launchpad to perform the sync request
-where possible.
+directly, which is the preferred method for uploads to the Ubuntu primary
+archive.
 .SH OPTIONS
 .TP
 \fB\-h\fR, \fB\-\-help\fR
@@ -43,13 +44,22 @@ Specify the component to sync from.
 \fB\-v\fR, \fB\-\-verbose\fR
 Display more progress information.
 .TP
+.B \-\-no\-lp
+Construct sync locally rather than letting Launchpad copy the package
+directly (not recommended).
+.TP
+\fB\-l\fI INSTANCE\fR, \fB\-\-lpinstance\fR=\fIINSTANCE\fR
+Launchpad instance to connect to (default: production).
+.TP
 \fB\-n\fI UPLOADER_NAME\fR, \fB\-\-uploader\-name\fR=\fIUPLOADER_NAME\fR
 Use UPLOADER_NAME as the name of the maintainer for this upload instead
 of evaluating DEBFULLNAME and UBUMAIL.
+This option may only be used in \fB\-\-no\-lp\fR mode.
 .TP
 \fB\-e\fI UPLOADER_EMAIL\fR, \fB\-\-uploader\-email\fR=\fIUPLOADER_EMAIL\fR
 Use UPLOADER_EMAIL as the email address of the maintainer for this
 upload instead of evaluating DEBEMAIL and UBUMAIL.
+This option may only be used in \fB\-\-no\-lp\fR mode.
 .TP
 \fB\-k\fI KEYID\fR, \fB\-\-key\fR=\fIKEYID\fR
 Specify the key ID to be used for signing.
@@ -60,6 +70,9 @@ Do not sign the upload.
 \fB\-b\fI BUG\fR, \fB\-\-bug\fR=\fIBUG\fR
 Mark a Launchpad bug as being fixed by this upload.
 .TP
+\fB\-f\fR, \fB\-\-force\fR
+Force sync over the top of Ubuntu changes.
+.TP
 .B \-d \fIDEBIAN_MIRROR\fR, \fB\-\-debian\-mirror\fR=\fIDEBIAN_MIRROR\fR
 Use the specified mirror.
 Should be in the form \fBhttp://ftp.debian.org/debian\fR.
@@ -75,6 +88,9 @@ back to the default mirror.
 .B \-\-no\-conf
 Do not read any configuration files, or configuration from environment
 variables.
+.TP
+.B \-\-simulate
+Show what would be done, but don't actually do it.
 .SH ENVIRONMENT
 .TP
 .BR DEBFULLNAME ", " DEBEMAIL ", " UBUMAIL

requestsync

@@ -239,6 +239,10 @@ def main():
     if not sponsorship:
         sponsorship = needSponsorship(srcpkg, ubuntu_component, release)
 
+    if not sponsorship and not ffe:
+        print >> sys.stderr, ('Consider using syncpackage(1) for syncs that '
+                              'do not require feature freeze exceptions.')
+
     # Check for existing package reports
     if not newsource:
         checkExistingReports(srcpkg)

syncpackage

@@ -29,6 +29,8 @@ import sys
 
 from devscripts.logger import Logger
 
+from lazr.restfulclient.errors import HTTPError
+
 from ubuntutools.archive import (DebianSourcePackage, UbuntuSourcePackage,
                                  DownloadError)
 from ubuntutools.config import UDTConfig, ubu_email
@@ -36,7 +38,9 @@ from ubuntutools.requestsync.mail import (getDebianSrcPkg
                                           as requestsync_mail_getDebianSrcPkg)
 from ubuntutools.requestsync.lp import getDebianSrcPkg, getUbuntuSrcPkg
 from ubuntutools.lp import udtexceptions
-from ubuntutools.lp.lpapicache import Launchpad
+from ubuntutools.lp.lpapicache import Distribution, Launchpad
+from ubuntutools.misc import split_release_pocket
+from ubuntutools.question import YesNoQuestion
 from ubuntutools import subprocess
 
 
@@ -110,7 +114,7 @@ def add_fixed_bugs(changes, bugs):
     return "\n".join(changes + [""])
 
 def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror,
-             keyid=None):
+             keyid=None, simulate=False, force=False):
     uploader = name + " <" + email + ">"
 
     src_pkg.pull_dsc()
@@ -136,9 +140,15 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror,
 
     cur_ver = ubuntu_ver.get_related_debian_version()
     if ubuntu_ver.is_modified_in_ubuntu():
+        if not force:
+            Logger.error('--force is required to discard Ubuntu changes.')
+            sys.exit(1)
+
         Logger.warn('Overwriting modified Ubuntu version %s, '
                     'setting current version to %s',
                     ubuntu_ver.full_version, cur_ver.full_version)
+    if simulate:
+        return
 
     try:
         src_pkg.pull()
@@ -248,8 +258,13 @@ def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror):
     """Download the specified source package.
     dist, version, component, mirror can all be None.
     """
+    if mirror is None:
+        mirrors = []
+    else:
+        mirrors = [mirror]
+
     if package.endswith('.dsc'):
-        return DebianSourcePackage(dscfile=package, mirrors=[mirror])
+        return DebianSourcePackage(dscfile=package, mirrors=mirrors)
 
     if dist is None:
         dist = "unstable"
@@ -290,7 +305,80 @@ def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror):
     assert component in ('main', 'contrib', 'non-free')
 
     return DebianSourcePackage(package, version.full_version, component,
-                               mirrors=[mirror])
+                               mirrors=mirrors)
+
+def copy(src_pkg, release, simulate=False, force=False):
+    """Copy a source package from Debian to Ubuntu using the Launchpad API."""
+    ubuntu = Distribution('ubuntu')
+    debian_archive = Distribution('debian').getArchive()
+    ubuntu_archive = ubuntu.getArchive()
+    if release is None:
+        ubuntu_series = ubuntu.getDevelopmentSeries().name
+        ubuntu_pocket = 'Release'
+    else:
+        ubuntu_series, ubuntu_pocket = split_release_pocket(release)
+
+    # Ensure that the provided Debian version actually exists.
+    try:
+        debian_archive.getPublishedSources(
+            source_name=src_pkg.source,
+            version=src_pkg.version.full_version,
+            exact_match=True)[0]
+    except IndexError:
+        Logger.error('Debian version %s does not exist!', src_pkg.version)
+        sys.exit(1)
+
+    try:
+        ubuntu_spph = getUbuntuSrcPkg(src_pkg.source,
+                                      ubuntu_series, ubuntu_pocket)
+        ubuntu_pkg = UbuntuSourcePackage(src_pkg.source,
+                                         ubuntu_spph.getVersion(),
+                                         ubuntu_spph.getComponent(),
+                                         mirrors=[])
+
+        Logger.normal('Source %s -> %s/%s: current version %s, new version %s',
+                      src_pkg.source, ubuntu_series, ubuntu_pocket,
+                      ubuntu_pkg.version, src_pkg.version)
+
+        ubuntu_version = Version(ubuntu_pkg.version.full_version)
+        if not force and ubuntu_version.is_modified_in_ubuntu():
+            Logger.error('--force is required to discard Ubuntu changes.')
+            sys.exit(1)
+
+        # Check whether a fakesync would be required.
+        src_pkg.pull_dsc()
+        ubuntu_pkg.pull_dsc()
+        if not src_pkg.dsc.compare_dsc(ubuntu_pkg.dsc):
+            Logger.error('The checksums of the Debian and Ubuntu packages '
+                         'mismatch. A fake sync using --no-lp is required.')
+            sys.exit(1)
+    except udtexceptions.PackageNotFoundException:
+        Logger.normal('Source %s -> %s/%s: not in Ubuntu, new version %s',
+                      src_pkg.source, ubuntu_series, ubuntu_pocket,
+                      src_pkg.version)
+    if simulate:
+        return
+
+    answer = YesNoQuestion().ask("Sync this package", "no")
+    if answer != "yes":
+        return
+
+    try:
+        ubuntu_archive.copyPackage(
+            source_name=src_pkg.source,
+            version=src_pkg.version.full_version,
+            from_archive=debian_archive,
+            to_series=ubuntu_series,
+            to_pocket=ubuntu_pocket,
+            include_binaries=False)
+    except HTTPError, error:
+        Logger.error("HTTP Error %s: %s", error.response.status,
+                     error.response.reason)
+        Logger.error(error.content)
+        sys.exit(1)
+
+    Logger.normal('Request succeeded; you should get an e-mail once it is '
+                  'processed.')
 
 def main():
     usage = "%prog [options] <.dsc URL/path or package name>"
@@ -312,6 +400,15 @@ def main():
     parser.add_option("-v", "--verbose",
                       dest="verbose", action="store_true", default=False,
                       help="Display more progress information.")
+    parser.add_option("--no-lp",
+                      dest="lp", action="store_false", default=True,
+                      help="Construct sync locally rather than letting "
+                           "Launchpad copy the package directly (not "
+                           "recommended).")
+    parser.add_option('-l', '--lpinstance', metavar='INSTANCE',
+                      dest='lpinstance', default=None,
+                      help='Launchpad instance to connect to '
+                           '(default: production).')
     parser.add_option("-n", "--uploader-name",
                       dest="uploader_name", default=None,
                       help="Use UPLOADER_NAME as the name of the maintainer "
@@ -330,6 +427,9 @@ def main():
                       dest="bugs", action="append", default=list(),
                       help="Mark Launchpad bug BUG as being fixed by this "
                            "upload.")
+    parser.add_option("-f", "--force",
+                      dest="force", action="store_true", default=False,
+                      help="Force sync over the top of Ubuntu changes.")
     parser.add_option('-D', '--debian-mirror', metavar='DEBIAN_MIRROR',
                       dest='debian_mirror',
                       help='Preferred Debian mirror '
@@ -343,6 +443,10 @@ def main():
     parser.add_option('--no-conf',
                       dest='no_conf', default=False, action='store_true',
                       help="Don't read config files or environment variables.")
+    parser.add_option('--simulate',
+                      dest='simulate', default=False, action='store_true',
+                      help="Show what would be done, but don't actually do "
+                           "it.")
 
     (options, args) = parser.parse_args()
 
@@ -362,6 +466,14 @@ def main():
                      'It should be one of main, contrib, or non-free.'
                      % options.component)
 
+    if options.lp and options.uploader_name:
+        parser.error('Uploader name can only be overridden using --no-lp.')
+    if options.lp and options.uploader_email:
+        parser.error('Uploader email address can only be overridden using '
+                     '--no-lp.')
+    # --key, --dont-sign, --debian-mirror, and --ubuntu-mirror are just
+    # ignored with options.lp, and do not require warnings.
+
     Logger.verbose = options.verbose
     config = UDTConfig(options.no_conf)
     if options.debian_mirror is None:
@@ -373,9 +485,27 @@ def main():
     if options.uploader_email is None:
         options.uploader_email = ubu_email(export=False)[1]
 
+    if options.lp:
+        if args[0].endswith('.dsc'):
+            parser.error('.dsc files can only be synced using --no-lp.')
+
+        if options.lpinstance is None:
+            options.lpinstance = config.get_value('LPINSTANCE')
+
+        try:
+            Launchpad.login(service=options.lpinstance, api_version='devel')
+        except IOError:
+            sys.exit(1)
+
+        src_pkg = fetch_source_pkg(args[0], options.dist, options.debversion,
+                                   options.component, options.release, None)
+
+        copy(src_pkg, options.release, options.simulate, options.force)
+    else:
         Launchpad.login_anonymously()
         if options.release is None:
-        options.release = Launchpad.distributions["ubuntu"].current_series.name
+            ubuntu = Launchpad.distributions["ubuntu"]
+            options.release = ubuntu.current_series.name
 
         os.environ['DEB_VENDOR'] = 'Ubuntu'
 
@@ -385,7 +515,7 @@ def main():
 
         sync_dsc(src_pkg, options.dist, options.release, options.uploader_name,
                  options.uploader_email, options.bugs, options.ubuntu_mirror,
-             options.keyid)
+                 options.keyid, options.simulate, options.force)
 
 if __name__ == "__main__":
     main()

ubuntutools/archive.py

@@ -95,6 +95,30 @@ class Dsc(debian.deb822.Dsc):
             return hash_func.hexdigest() == digest
         return False
 
+    def compare_dsc(self, other):
+        """Check whether any files in these two dscs that have the same name
+        also have the same checksum."""
+        for field, key in (('Checksums-Sha256', 'sha256'),
+                           ('Checksums-Sha1', 'sha1'),
+                           ('Files', 'md5sum')):
+            if field not in self or field not in other:
+                continue
+            our_checksums = \
+                dict((entry['name'], (int(entry['size']), entry[key]))
+                     for entry in self[field])
+            their_checksums = \
+                dict((entry['name'], (int(entry['size']), entry[key]))
+                     for entry in other[field])
+            for name, (size, checksum) in our_checksums.iteritems():
+                if name not in their_checksums:
+                    # file only in one dsc
+                    continue
+                if (size != their_checksums[name][0] or
+                    checksum != their_checksums[name][1]):
+                    return False
+            return True # one checksum is good enough
+        return True
+
 
 class SourcePackage(object):
     """Base class for source package downloading.

ubuntutools/lp/lpapicache.py

@@ -53,7 +53,7 @@ _POCKETS = ('Release', 'Security', 'Updates', 'Proposed', 'Backports')
 class _Launchpad(object):
     '''Singleton for LP API access.'''
 
-    def login(self, service=service):
+    def login(self, service=service, api_version=api_version):
         '''Enforce a non-anonymous login.'''
         if not self.logged_in:
             try:
@@ -332,6 +332,25 @@ class Archive(BaseWrapper):
 
         return self._srcpkgs[(name, series.name, pocket)]
 
+    def copyPackage(self, source_name, version, from_archive, to_pocket,
+                    to_series = None, include_binaries = False):
+        '''Copy a single named source into this archive.
+
+        Asynchronously copy a specific version of a named source to the
+        destination archive if necessary.  Calls to this method will return
+        immediately if the copy passes basic security checks and the copy
+        will happen sometime later with full checking.
+        '''
+
+        self._lpobject.copyPackage(
+            source_name=source_name,
+            version=version,
+            from_archive=from_archive._lpobject,
+            to_pocket=to_pocket,
+            to_series=to_series,
+            include_binaries=include_binaries
+            )
+
 
 class SourcePackagePublishingHistory(BaseWrapper):
     '''

ubuntutools/requestsync/lp.py

@@ -34,11 +34,11 @@ def getDebianSrcPkg(name, release):
 
     return debian_archive.getSourcePackage(name, release)
 
-def getUbuntuSrcPkg(name, release):
+def getUbuntuSrcPkg(name, release, pocket='Release'):
     ubuntu = Distribution('ubuntu')
     ubuntu_archive = ubuntu.getArchive()
 
-    return ubuntu_archive.getSourcePackage(name, release)
+    return ubuntu_archive.getSourcePackage(name, release, pocket)
 
 def needSponsorship(name, component, release):
     '''