diff --git a/debian/changelog b/debian/changelog index 302af1b..8a77a29 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +ubuntu-dev-tools (0.129) UNRELEASED; urgency=low + + * syncpackage: Convert to new LP API, with --no-lp available for the old + style of operation. + * syncpackage: Require -f/--force option to overwrite Ubuntu changes. + + -- Colin Watson Tue, 16 Aug 2011 16:40:22 +0100 + ubuntu-dev-tools (0.128) unstable; urgency=low [ Stefano Rivera ] diff --git a/doc/requestsync.1 b/doc/requestsync.1 index 67bde76..702cd70 100644 --- a/doc/requestsync.1 +++ b/doc/requestsync.1 @@ -32,6 +32,11 @@ This check is only performed if \fBrequestsync\fR is allowed to use the LP API answer the question about upload permissions honestly to determine if a team with approval rights is to be subscribed to the bug. +If you have permission to upload the package directly, then you may prefer +to use \fBsyncpackage\fR instead to copy the package using the Launchpad +API. At some future point, \fBrequestsync\fR will be changed to do this +automatically. + .PP \fBrequestsync\fR uses launchpadlib authentication to file its requests. diff --git a/doc/syncpackage.1 b/doc/syncpackage.1 index 22f10dc..009f654 100644 --- a/doc/syncpackage.1 +++ b/doc/syncpackage.1 @@ -1,12 +1,12 @@ .TH SYNCPACKAGE "1" "June 2010" "ubuntu-dev-tools" .SH NAME -syncpackage \- helper to prepare .changes file to upload synced packages +syncpackage \- copy source packages from Debian to Ubuntu .SH SYNOPSIS .B syncpackage [\fIoptions\fR] \fI<.dsc URL/path or package name>\fR .SH DESCRIPTION -\fBsyncpackage\fR generates a changes file to be directly uploaded to Ubuntu -primary archive or PPA starting from a pristine Debian package. +\fBsyncpackage\fR causes a source package to be copied from Debian to +Ubuntu. .PP \fBsyncpackage\fR allows you to upload files with the same checksums of the Debian ones, as the common script used by Ubuntu archive administrators does, @@ -15,14 +15,15 @@ this way you can preserve source files integrity between the two distributions. \fBsyncpackage\fR will detect source tarballs with mismatching checksums and will automatically create fake syncs instead. .SH WARNING -The use of \fBsyncpackage\fR is discouraged by the Ubuntu Archive -Administrators, as it introduces an unnecessary window for error. -In future launchpad will offer the ability to directly perform sync -requests, without a developer having to do an upload. -Until then, it's advised to use \fBsyncpackage\fR with caution, and -request syncs via bugs with -.BR requestsync (1) -where possible. +The use of \fBsyncpackage \-\-no\-lp\fR, which generates a changes file to +be directly uploaded to the Ubuntu primary archive or a PPA, is discouraged +by the Ubuntu Archive Administrators, as it introduces an unnecessary window +for error. +This only exists for backward compatibility, for unusual corner cases, and +for uploads to archives other than the Ubuntu primary archive. +Omitting this option will cause Launchpad to perform the sync request +directly, which is the preferred method for uploads to the Ubuntu primary +archive. .SH OPTIONS .TP \fB\-h\fR, \fB\-\-help\fR @@ -43,13 +44,22 @@ Specify the component to sync from. \fB\-v\fR, \fB\-\-verbose\fR Display more progress information. .TP +.B \-\-no\-lp +Construct sync locally rather than letting Launchpad copy the package +directly (not recommended). +.TP +\fB\-l\fI INSTANCE\fR, \fB\-\-lpinstance\fR=\fIINSTANCE\fR +Launchpad instance to connect to (default: production). +.TP \fB\-n\fI UPLOADER_NAME\fR, \fB\-\-uploader\-name\fR=\fIUPLOADER_NAME\fR Use UPLOADER_NAME as the name of the maintainer for this upload instead of evaluating DEBFULLNAME and UBUMAIL. +This option may only be used in \fB\-\-no\-lp\fR mode. .TP \fB\-e\fI UPLOADER_EMAIL\fR, \fB\-\-uploader\-email\fR=\fIUPLOADER_EMAIL\fR Use UPLOADER_EMAIL as the email address of the maintainer for this upload instead of evaluating DEBEMAIL and UBUMAIL. +This option may only be used in \fB\-\-no\-lp\fR mode. .TP \fB\-k\fI KEYID\fR, \fB\-\-key\fR=\fIKEYID\fR Specify the key ID to be used for signing. @@ -60,6 +70,9 @@ Do not sign the upload. \fB\-b\fI BUG\fR, \fB\-\-bug\fR=\fIBUG\fR Mark a Launchpad bug as being fixed by this upload. .TP +\fB\-f\fR, \fB\-\-force\fR +Force sync over the top of Ubuntu changes. +.TP .B \-d \fIDEBIAN_MIRROR\fR, \fB\-\-debian\-mirror\fR=\fIDEBIAN_MIRROR\fR Use the specified mirror. Should be in the form \fBhttp://ftp.debian.org/debian\fR. @@ -75,6 +88,9 @@ back to the default mirror. .B \-\-no\-conf Do not read any configuration files, or configuration from environment variables. +.TP +.B \-\-simulate +Show what would be done, but don't actually do it. .SH ENVIRONMENT .TP .BR DEBFULLNAME ", " DEBEMAIL ", " UBUMAIL diff --git a/requestsync b/requestsync index 9b3ff59..33a6462 100755 --- a/requestsync +++ b/requestsync @@ -239,6 +239,10 @@ def main(): if not sponsorship: sponsorship = needSponsorship(srcpkg, ubuntu_component, release) + if not sponsorship and not ffe: + print >> sys.stderr, ('Consider using syncpackage(1) for syncs that ' + 'do not require feature freeze exceptions.') + # Check for existing package reports if not newsource: checkExistingReports(srcpkg) diff --git a/syncpackage b/syncpackage index a47a182..ee4225c 100755 --- a/syncpackage +++ b/syncpackage @@ -29,6 +29,8 @@ import sys from devscripts.logger import Logger +from lazr.restfulclient.errors import HTTPError + from ubuntutools.archive import (DebianSourcePackage, UbuntuSourcePackage, DownloadError) from ubuntutools.config import UDTConfig, ubu_email @@ -36,7 +38,9 @@ from ubuntutools.requestsync.mail import (getDebianSrcPkg as requestsync_mail_getDebianSrcPkg) from ubuntutools.requestsync.lp import getDebianSrcPkg, getUbuntuSrcPkg from ubuntutools.lp import udtexceptions -from ubuntutools.lp.lpapicache import Launchpad +from ubuntutools.lp.lpapicache import Distribution, Launchpad +from ubuntutools.misc import split_release_pocket +from ubuntutools.question import YesNoQuestion from ubuntutools import subprocess @@ -110,7 +114,7 @@ def add_fixed_bugs(changes, bugs): return "\n".join(changes + [""]) def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, - keyid=None): + keyid=None, simulate=False, force=False): uploader = name + " <" + email + ">" src_pkg.pull_dsc() @@ -136,9 +140,15 @@ def sync_dsc(src_pkg, debian_dist, release, name, email, bugs, ubuntu_mirror, cur_ver = ubuntu_ver.get_related_debian_version() if ubuntu_ver.is_modified_in_ubuntu(): + if not force: + Logger.error('--force is required to discard Ubuntu changes.') + sys.exit(1) + Logger.warn('Overwriting modified Ubuntu version %s, ' 'setting current version to %s', ubuntu_ver.full_version, cur_ver.full_version) + if simulate: + return try: src_pkg.pull() @@ -248,8 +258,13 @@ def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror): """Download the specified source package. dist, version, component, mirror can all be None. """ + if mirror is None: + mirrors = [] + else: + mirrors = [mirror] + if package.endswith('.dsc'): - return DebianSourcePackage(dscfile=package, mirrors=[mirror]) + return DebianSourcePackage(dscfile=package, mirrors=mirrors) if dist is None: dist = "unstable" @@ -290,7 +305,80 @@ def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror): assert component in ('main', 'contrib', 'non-free') return DebianSourcePackage(package, version.full_version, component, - mirrors=[mirror]) + mirrors=mirrors) + +def copy(src_pkg, release, simulate=False, force=False): + """Copy a source package from Debian to Ubuntu using the Launchpad API.""" + ubuntu = Distribution('ubuntu') + debian_archive = Distribution('debian').getArchive() + ubuntu_archive = ubuntu.getArchive() + if release is None: + ubuntu_series = ubuntu.getDevelopmentSeries().name + ubuntu_pocket = 'Release' + else: + ubuntu_series, ubuntu_pocket = split_release_pocket(release) + + # Ensure that the provided Debian version actually exists. + try: + debian_archive.getPublishedSources( + source_name=src_pkg.source, + version=src_pkg.version.full_version, + exact_match=True)[0] + except IndexError: + Logger.error('Debian version %s does not exist!', src_pkg.version) + sys.exit(1) + + try: + ubuntu_spph = getUbuntuSrcPkg(src_pkg.source, + ubuntu_series, ubuntu_pocket) + ubuntu_pkg = UbuntuSourcePackage(src_pkg.source, + ubuntu_spph.getVersion(), + ubuntu_spph.getComponent(), + mirrors=[]) + + Logger.normal('Source %s -> %s/%s: current version %s, new version %s', + src_pkg.source, ubuntu_series, ubuntu_pocket, + ubuntu_pkg.version, src_pkg.version) + + ubuntu_version = Version(ubuntu_pkg.version.full_version) + if not force and ubuntu_version.is_modified_in_ubuntu(): + Logger.error('--force is required to discard Ubuntu changes.') + sys.exit(1) + + # Check whether a fakesync would be required. + src_pkg.pull_dsc() + ubuntu_pkg.pull_dsc() + if not src_pkg.dsc.compare_dsc(ubuntu_pkg.dsc): + Logger.error('The checksums of the Debian and Ubuntu packages ' + 'mismatch. A fake sync using --no-lp is required.') + sys.exit(1) + except udtexceptions.PackageNotFoundException: + Logger.normal('Source %s -> %s/%s: not in Ubuntu, new version %s', + src_pkg.source, ubuntu_series, ubuntu_pocket, + src_pkg.version) + if simulate: + return + + answer = YesNoQuestion().ask("Sync this package", "no") + if answer != "yes": + return + + try: + ubuntu_archive.copyPackage( + source_name=src_pkg.source, + version=src_pkg.version.full_version, + from_archive=debian_archive, + to_series=ubuntu_series, + to_pocket=ubuntu_pocket, + include_binaries=False) + except HTTPError, error: + Logger.error("HTTP Error %s: %s", error.response.status, + error.response.reason) + Logger.error(error.content) + sys.exit(1) + + Logger.normal('Request succeeded; you should get an e-mail once it is ' + 'processed.') def main(): usage = "%prog [options] <.dsc URL/path or package name>" @@ -312,6 +400,15 @@ def main(): parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, help="Display more progress information.") + parser.add_option("--no-lp", + dest="lp", action="store_false", default=True, + help="Construct sync locally rather than letting " + "Launchpad copy the package directly (not " + "recommended).") + parser.add_option('-l', '--lpinstance', metavar='INSTANCE', + dest='lpinstance', default=None, + help='Launchpad instance to connect to ' + '(default: production).') parser.add_option("-n", "--uploader-name", dest="uploader_name", default=None, help="Use UPLOADER_NAME as the name of the maintainer " @@ -330,6 +427,9 @@ def main(): dest="bugs", action="append", default=list(), help="Mark Launchpad bug BUG as being fixed by this " "upload.") + parser.add_option("-f", "--force", + dest="force", action="store_true", default=False, + help="Force sync over the top of Ubuntu changes.") parser.add_option('-D', '--debian-mirror', metavar='DEBIAN_MIRROR', dest='debian_mirror', help='Preferred Debian mirror ' @@ -343,6 +443,10 @@ def main(): parser.add_option('--no-conf', dest='no_conf', default=False, action='store_true', help="Don't read config files or environment variables.") + parser.add_option('--simulate', + dest='simulate', default=False, action='store_true', + help="Show what would be done, but don't actually do " + "it.") (options, args) = parser.parse_args() @@ -362,6 +466,14 @@ def main(): 'It should be one of main, contrib, or non-free.' % options.component) + if options.lp and options.uploader_name: + parser.error('Uploader name can only be overridden using --no-lp.') + if options.lp and options.uploader_email: + parser.error('Uploader email address can only be overridden using ' + '--no-lp.') + # --key, --dont-sign, --debian-mirror, and --ubuntu-mirror are just + # ignored with options.lp, and do not require warnings. + Logger.verbose = options.verbose config = UDTConfig(options.no_conf) if options.debian_mirror is None: @@ -373,19 +485,37 @@ def main(): if options.uploader_email is None: options.uploader_email = ubu_email(export=False)[1] - Launchpad.login_anonymously() - if options.release is None: - options.release = Launchpad.distributions["ubuntu"].current_series.name + if options.lp: + if args[0].endswith('.dsc'): + parser.error('.dsc files can only be synced using --no-lp.') - os.environ['DEB_VENDOR'] = 'Ubuntu' + if options.lpinstance is None: + options.lpinstance = config.get_value('LPINSTANCE') - src_pkg = fetch_source_pkg(args[0], options.dist, options.debversion, - options.component, options.release, - options.debian_mirror) + try: + Launchpad.login(service=options.lpinstance, api_version='devel') + except IOError: + sys.exit(1) - sync_dsc(src_pkg, options.dist, options.release, options.uploader_name, - options.uploader_email, options.bugs, options.ubuntu_mirror, - options.keyid) + src_pkg = fetch_source_pkg(args[0], options.dist, options.debversion, + options.component, options.release, None) + + copy(src_pkg, options.release, options.simulate, options.force) + else: + Launchpad.login_anonymously() + if options.release is None: + ubuntu = Launchpad.distributions["ubuntu"] + options.release = ubuntu.current_series.name + + os.environ['DEB_VENDOR'] = 'Ubuntu' + + src_pkg = fetch_source_pkg(args[0], options.dist, options.debversion, + options.component, options.release, + options.debian_mirror) + + sync_dsc(src_pkg, options.dist, options.release, options.uploader_name, + options.uploader_email, options.bugs, options.ubuntu_mirror, + options.keyid, options.simulate, options.force) if __name__ == "__main__": main() diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index f54c4dd..4647615 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -95,6 +95,30 @@ class Dsc(debian.deb822.Dsc): return hash_func.hexdigest() == digest return False + def compare_dsc(self, other): + """Check whether any files in these two dscs that have the same name + also have the same checksum.""" + for field, key in (('Checksums-Sha256', 'sha256'), + ('Checksums-Sha1', 'sha1'), + ('Files', 'md5sum')): + if field not in self or field not in other: + continue + our_checksums = \ + dict((entry['name'], (int(entry['size']), entry[key])) + for entry in self[field]) + their_checksums = \ + dict((entry['name'], (int(entry['size']), entry[key])) + for entry in other[field]) + for name, (size, checksum) in our_checksums.iteritems(): + if name not in their_checksums: + # file only in one dsc + continue + if (size != their_checksums[name][0] or + checksum != their_checksums[name][1]): + return False + return True # one checksum is good enough + return True + class SourcePackage(object): """Base class for source package downloading. diff --git a/ubuntutools/lp/lpapicache.py b/ubuntutools/lp/lpapicache.py index 097b63f..b7339ca 100644 --- a/ubuntutools/lp/lpapicache.py +++ b/ubuntutools/lp/lpapicache.py @@ -53,7 +53,7 @@ _POCKETS = ('Release', 'Security', 'Updates', 'Proposed', 'Backports') class _Launchpad(object): '''Singleton for LP API access.''' - def login(self, service=service): + def login(self, service=service, api_version=api_version): '''Enforce a non-anonymous login.''' if not self.logged_in: try: @@ -332,6 +332,25 @@ class Archive(BaseWrapper): return self._srcpkgs[(name, series.name, pocket)] + def copyPackage(self, source_name, version, from_archive, to_pocket, + to_series = None, include_binaries = False): + '''Copy a single named source into this archive. + + Asynchronously copy a specific version of a named source to the + destination archive if necessary. Calls to this method will return + immediately if the copy passes basic security checks and the copy + will happen sometime later with full checking. + ''' + + self._lpobject.copyPackage( + source_name=source_name, + version=version, + from_archive=from_archive._lpobject, + to_pocket=to_pocket, + to_series=to_series, + include_binaries=include_binaries + ) + class SourcePackagePublishingHistory(BaseWrapper): ''' diff --git a/ubuntutools/requestsync/lp.py b/ubuntutools/requestsync/lp.py index 60ce01f..0165161 100644 --- a/ubuntutools/requestsync/lp.py +++ b/ubuntutools/requestsync/lp.py @@ -34,11 +34,11 @@ def getDebianSrcPkg(name, release): return debian_archive.getSourcePackage(name, release) -def getUbuntuSrcPkg(name, release): +def getUbuntuSrcPkg(name, release, pocket='Release'): ubuntu = Distribution('ubuntu') ubuntu_archive = ubuntu.getArchive() - return ubuntu_archive.getSourcePackage(name, release) + return ubuntu_archive.getSourcePackage(name, release, pocket) def needSponsorship(name, component, release): '''