From b01194ef8db247aadb9a8a6d0516b839982e1496 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Mon, 13 Dec 2010 03:47:37 -0800 Subject: [PATCH 01/10] ubuntutools.builder: Add an "update" method for updating chroots --- ubuntutools/builder.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/ubuntutools/builder.py b/ubuntutools/builder.py index e15a534..e0687dc 100644 --- a/ubuntutools/builder.py +++ b/ubuntutools/builder.py @@ -50,6 +50,11 @@ class Pbuilder(Builder): Logger.command(cmd) return subprocess.call(cmd) + def update(self, dist): + cmd = ["sudo", "-E", "DIST=" + dist, "pbuilder", "--update"] + Logger.command(cmd) + return subprocess.call(cmd) + class Sbuild(Builder): def __init__(self): @@ -67,6 +72,38 @@ class Sbuild(Builder): os.chdir(workdir) return result + def update(self, dist): + cmd = ["schroot", "--list"] + Logger.command(cmd) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + chroots, _ = p.communicate() + chroots = chroots.strip().split() + if p.returncode != 0: + return p.returncode + + params = {"dist": dist, + "arch": self.architecture} + for chroot in ("%(dist)s-%(arch)s-sbuild-source", + "%(dist)s-sbuild-source", + "%(dist)s-%(arch)s-source", + "%(dist)s-source"): + chroot = chroot % params + if chroot in chroots: + break + else: + return 1 + + chroot_cmd = ["sudo", "schroot", "-c", chroot, "-u", "root", "--"] + commands = [["apt-get", "-q", "update"], + ["apt-get", "-q", "-y", "dist-upgrade"], + ["apt-get", "-q", "-y", "autoremove"], + ["apt-get", "-q", "clean"]] + for cmd in commands: + Logger.command(chroot_cmd + cmd) + ret = subprocess.call(chroot_cmd + cmd) + if ret != 0: + return ret + def getBuilder(builder=None): if not builder: From 9f6067b68f7da32abab1d7d23e4f5935a6aaf877 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Mon, 13 Dec 2010 04:00:14 -0800 Subject: [PATCH 02/10] sponsor-patch: Add --update option to make sure build environment is up to date (LP: #689605) --- backportpackage | 8 ++++++++ debian/changelog | 4 +++- doc/backportpackage.1 | 3 +++ doc/sponsor-patch.1 | 3 +++ sponsor-patch | 23 ++++++++++++++++++----- 5 files changed, 35 insertions(+), 6 deletions(-) diff --git a/backportpackage b/backportpackage index 27eebed..8a1411c 100755 --- a/backportpackage +++ b/backportpackage @@ -65,6 +65,11 @@ def parse(args): default=None, help='Specify the package builder (default: pbuilder)', metavar='BUILDER') + p.add_option('-U', '--update', + dest='update', + default=False, + action='store_true', + help='Update the build environment before attempting to build') p.add_option('-u', '--upload', dest='upload', help='Specify an upload destination', @@ -166,6 +171,9 @@ def do_build(workdir, package, release, bp_version, opts): if not builder: return + if opts.update: + builder.update(release) + builder.build(os.path.join(workdir, '%s_%s.dsc' % (package, bp_version)), release, diff --git a/debian/changelog b/debian/changelog index 3cc0184..e425cf6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,8 +7,10 @@ ubuntu-dev-tools (0.108) UNRELEASED; urgency=low [ Evan Broder ] * backportpackage: new script for testing backport requests in a PPA. + * sponsor-patch: Add --update option to make sure build environment is + up to date (LP: #689605) - -- Evan Broder Sat, 11 Dec 2010 14:11:54 -0800 + -- Evan Broder Mon, 13 Dec 2010 03:57:20 -0800 ubuntu-dev-tools (0.107) experimental; urgency=low diff --git a/doc/backportpackage.1 b/doc/backportpackage.1 index 341beaf..8fce4c7 100644 --- a/doc/backportpackage.1 +++ b/doc/backportpackage.1 @@ -35,6 +35,9 @@ Use the specified builder to build the package. Supported are \fBpbuilder\fR(8) and \fBsbuild\fR(1). This overrides \fBUBUNTUTOOLS_BUILDER\fR. The default is \fBpbuilder\fR(8). .TP +.B \-U, \-\-update +Update the builder environment before attempting to build. +.TP .B \-u \fIUPLOAD\fR, \-\-upload=\fIUPLOAD\fR Upload to \fIUPLOAD\fR with \fBdput\fR(1) (after confirmation). .TP diff --git a/doc/sponsor-patch.1 b/doc/sponsor-patch.1 index c0e5b6b..5c4f1ec 100644 --- a/doc/sponsor-patch.1 +++ b/doc/sponsor-patch.1 @@ -77,6 +77,9 @@ Shortcut for sponsored uploads. Equivalent to \fB\-b \-u ubuntu\fR. .B \-u \fIDEST\fR, \fB\-\-upload\fR=\fIDEST Upload to \fIDEST\fR with \fBdput\fR(1) (after confirmation). .TP +.BR \-U ", " \-\-update +Make sure the build environment is up to date before building. +.TP .BR \-v ", " \-\-verbose Print more information. .TP diff --git a/sponsor-patch b/sponsor-patch index 0d8cde6..63099a1 100755 --- a/sponsor-patch +++ b/sponsor-patch @@ -409,8 +409,8 @@ def apply_patch(task, patch): edit = True return edit -def main(script_name, bug_number, build, edit, keyid, upload, workdir, builder, - verbose=False): +def main(script_name, bug_number, update, build, edit, keyid, upload, workdir, + builder, verbose=False): workdir = os.path.expanduser(workdir) if not os.path.isdir(workdir): try: @@ -598,6 +598,15 @@ def main(script_name, bug_number, build, edit, keyid, upload, workdir, builder, continue if build: + if update: + ret = builder.update() + if ret != 0: + Logger.error("Failed to update %s chroot for %s." % \ + (changelog.distributions, + builder.get_name())) + ask_for_manual_fixing() + continue + buildresult = os.path.join(workdir, task.package + "-buildresult") if not os.path.isdir(buildresult): os.makedirs(buildresult) @@ -609,7 +618,7 @@ def main(script_name, bug_number, build, edit, keyid, upload, workdir, builder, Logger.error("Failed to build %s from source with %s." % \ (os.path.basename(new_dsc_file), builder.get_name())) - # TODO: Add "retry" and "update" option + # TODO: Add "retry" option ask_for_manual_fixing() continue @@ -706,6 +715,9 @@ if __name__ == "__main__": dest="sponsoring", action="store_true", default=False) parser.add_option("-u", "--upload", dest="upload", default=None, help="Specify an upload destination (default none).") + parser.add_option("-U", "--update", dest="update", default=False, + action="store_true", + help="Update builder chroot before building") parser.add_option("-v", "--verbose", help="print more information", dest="verbose", action="store_true", default=False) parser.add_option("-w", "--workdir", dest="workdir", @@ -737,5 +749,6 @@ if __name__ == "__main__": options.build = True options.upload = "ubuntu" - main(script_name, bug_number, options.build, options.edit, options.keyid, - options.upload, options.workdir, builder, options.verbose) + main(script_name, bug_number, options.update, options.build, options.edit, + options.keyid, options.upload, options.workdir, builder, + options.verbose) From ac1260b163837a9a184d3b6ec20b9e76cb22248e Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Mon, 13 Dec 2010 10:09:21 -0800 Subject: [PATCH 03/10] sponsor-patch: Replace accidentally removed comment --- sponsor-patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sponsor-patch b/sponsor-patch index 63099a1..185b597 100755 --- a/sponsor-patch +++ b/sponsor-patch @@ -541,7 +541,7 @@ def main(script_name, bug_number, update, build, edit, keyid, upload, workdir, Logger.command(cmd) if subprocess.call(cmd, env=env) != 0: Logger.error("Failed to build source tarball.") - # TODO: Add a "retry" option + # TODO: Add "retry" and "update" option ask_for_manual_fixing() continue From 6961cbfb3cd392394aa988f5554bcec97cc35903 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Mon, 13 Dec 2010 10:47:26 -0800 Subject: [PATCH 04/10] ubuntutools.builder: Use sbuild-update and friends instead of calling schroot directly --- ubuntutools/builder.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ubuntutools/builder.py b/ubuntutools/builder.py index e0687dc..f136eb8 100644 --- a/ubuntutools/builder.py +++ b/ubuntutools/builder.py @@ -93,14 +93,12 @@ class Sbuild(Builder): else: return 1 - chroot_cmd = ["sudo", "schroot", "-c", chroot, "-u", "root", "--"] - commands = [["apt-get", "-q", "update"], - ["apt-get", "-q", "-y", "dist-upgrade"], - ["apt-get", "-q", "-y", "autoremove"], - ["apt-get", "-q", "clean"]] + commands = [["sbuild-update"], + ["sbuild-distupgrade"], + ["sbuild-clean", "-a", "-c"]] for cmd in commands: - Logger.command(chroot_cmd + cmd) - ret = subprocess.call(chroot_cmd + cmd) + Logger.command(cmd + [chroot]) + ret = subprocess.call(cmd + [chroot]) if ret != 0: return ret From 5e181b6b8e23b8c3d0fee1e144ee835b6bcbc999 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Fri, 17 Dec 2010 01:36:05 -0800 Subject: [PATCH 05/10] backportpackage: Error out of builder update fails --- backportpackage | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backportpackage b/backportpackage index 4120280..a2834e9 100755 --- a/backportpackage +++ b/backportpackage @@ -193,7 +193,9 @@ def do_build(workdir, package, release, bp_version, builder): return if opts.update: - builder.update(release) + if 0 != builder.update(release): + error('Failed to update %s chroot for %s.' % \ + (release, builder.get_name())) return builder.build(os.path.join(workdir, '%s_%s.dsc' % (package, bp_version)), From 552b0a1f91ebe808f246d5533af4ea77794b3966 Mon Sep 17 00:00:00 2001 From: Evan Broder Date: Fri, 17 Dec 2010 01:48:45 -0800 Subject: [PATCH 06/10] Undo some accidental changes --- sponsor-patch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sponsor-patch b/sponsor-patch index 185b597..1ccfbfa 100755 --- a/sponsor-patch +++ b/sponsor-patch @@ -541,7 +541,7 @@ def main(script_name, bug_number, update, build, edit, keyid, upload, workdir, Logger.command(cmd) if subprocess.call(cmd, env=env) != 0: Logger.error("Failed to build source tarball.") - # TODO: Add "retry" and "update" option + # TODO: Add a "retry" option ask_for_manual_fixing() continue @@ -618,7 +618,7 @@ def main(script_name, bug_number, update, build, edit, keyid, upload, workdir, Logger.error("Failed to build %s from source with %s." % \ (os.path.basename(new_dsc_file), builder.get_name())) - # TODO: Add "retry" option + # TODO: Add "retry" and "update" option ask_for_manual_fixing() continue From 409762e55c4481042a2a3b630e42a5ce91133827 Mon Sep 17 00:00:00 2001 From: Benjamin Drung Date: Fri, 17 Dec 2010 11:46:33 +0100 Subject: [PATCH 07/10] Make command line flag description more consistent. --- doc/backportpackage.1 | 2 +- doc/sponsor-patch.1 | 2 +- sponsor-patch | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/backportpackage.1 b/doc/backportpackage.1 index 7d8edbd..5be1f1c 100644 --- a/doc/backportpackage.1 +++ b/doc/backportpackage.1 @@ -34,7 +34,7 @@ Use the specified builder to build the package. Supported are \fBUBUNTUTOOLS_BUILDER\fR. The default is \fBpbuilder\fR(8). .TP .B \-U, \-\-update -Update the builder environment before attempting to build. +Update the build environment before attempting to build. .TP .B \-u \fIUPLOAD\fR, \-\-upload=\fIUPLOAD\fR Upload to \fIUPLOAD\fR with \fBdput\fR(1) (after confirmation). diff --git a/doc/sponsor-patch.1 b/doc/sponsor-patch.1 index eee6ac7..dfb5355 100644 --- a/doc/sponsor-patch.1 +++ b/doc/sponsor-patch.1 @@ -78,7 +78,7 @@ Shortcut for sponsored uploads. Equivalent to \fB\-b \-u ubuntu\fR. Upload to \fIDEST\fR with \fBdput\fR(1) (after confirmation). .TP .BR \-U ", " \-\-update -Make sure the build environment is up to date before building. +Update the build environment before attempting to build. .TP .BR \-v ", " \-\-verbose Print more information. diff --git a/sponsor-patch b/sponsor-patch index 1ccfbfa..cfb907d 100755 --- a/sponsor-patch +++ b/sponsor-patch @@ -717,7 +717,7 @@ if __name__ == "__main__": help="Specify an upload destination (default none).") parser.add_option("-U", "--update", dest="update", default=False, action="store_true", - help="Update builder chroot before building") + help="Update the build environment before building.") parser.add_option("-v", "--verbose", help="print more information", dest="verbose", action="store_true", default=False) parser.add_option("-w", "--workdir", dest="workdir", From ec60ffc00e9b51c7c1966055de8259dbd2ba83a7 Mon Sep 17 00:00:00 2001 From: Benjamin Drung Date: Fri, 17 Dec 2010 11:49:22 +0100 Subject: [PATCH 08/10] sponsor-patch: Fix builder update logic. --- sponsor-patch | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sponsor-patch b/sponsor-patch index cfb907d..7c2612a 100755 --- a/sponsor-patch +++ b/sponsor-patch @@ -598,21 +598,24 @@ def main(script_name, bug_number, update, build, edit, keyid, upload, workdir, continue if build: + dist = re.sub("-.*$", "", changelog.distributions) + if update: - ret = builder.update() + ret = builder.update(dist) if ret != 0: Logger.error("Failed to update %s chroot for %s." % \ - (changelog.distributions, - builder.get_name())) + (dist, builder.get_name())) ask_for_manual_fixing() continue + # We want to update the build environment only once, but not + # after every manual fix. + update = False buildresult = os.path.join(workdir, task.package + "-buildresult") if not os.path.isdir(buildresult): os.makedirs(buildresult) # build package - dist = re.sub("-.*$", "", changelog.distributions) result = builder.build(new_dsc_file, dist, buildresult) if result != 0: Logger.error("Failed to build %s from source with %s." % \ From 9bbcd92ce4bf8dcbe7a336df6a90ba5f4e3b4add Mon Sep 17 00:00:00 2001 From: Benjamin Drung Date: Fri, 17 Dec 2010 13:48:05 +0100 Subject: [PATCH 09/10] Indentation fix. --- backportpackage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backportpackage b/backportpackage index 72fe94e..4b78200 100755 --- a/backportpackage +++ b/backportpackage @@ -190,7 +190,7 @@ def do_build(workdir, package, release, bp_version, builder): if opts.update: if 0 != builder.update(release): error('Failed to update %s chroot for %s.' % \ - (release, builder.get_name())) + (release, builder.get_name())) return builder.build(os.path.join(workdir, '%s_%s.dsc' % (package, bp_version)), From 7e038457bc58e027a585728a3d26ac1b4abd00bc Mon Sep 17 00:00:00 2001 From: Benjamin Drung Date: Fri, 17 Dec 2010 14:32:03 +0100 Subject: [PATCH 10/10] ubuntutools/builder.py: Pass distribution and architecture to pbuilder update function. --- ubuntutools/builder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ubuntutools/builder.py b/ubuntutools/builder.py index f136eb8..3449a86 100644 --- a/ubuntutools/builder.py +++ b/ubuntutools/builder.py @@ -51,7 +51,8 @@ class Pbuilder(Builder): return subprocess.call(cmd) def update(self, dist): - cmd = ["sudo", "-E", "DIST=" + dist, "pbuilder", "--update"] + cmd = ["sudo", "-E", "DIST=" + dist, "pbuilder", "--update", + "--distribution", dist, "--architecture", self.architecture] Logger.command(cmd) return subprocess.call(cmd)