Upload to Unstable

mk-sbuild installs pkgbinarymangler into the schroot.  Of of the
provided tools in pkgbinarymangler is pkgmaintainermangler.
pkgmaintainermangler is disabled by default, and enabled with
configuration.

A difference between launchpad builds of a synced package and an sbuild
is that the maintainer information will be different.

Enable pkgmaintainermangler to close this difference.

debian/.gitignore

@@ -0,0 +1 @@
+files

debian/changelog

@@ -1,3 +1,40 @@
+ubuntu-dev-tools (0.206) unstable; urgency=medium
+
+  [ Dan Bungert ]
+  * mk-sbuild: enable pkgmaintainermangler
+
+  [ Shengjing Zhu ]
+  * import-bug-from-debian: package option is overridden and not used
+
+  [ Fernando Bravo Hernández ]
+  * Parsing arch parameter to getBinaryPackage() (LP: #2081861)
+
+  [ Simon Quigley ]
+  * Read ~/.devscripts in a more robust way, to ideally pick up multi-line
+    variables (Closes: #725418).
+  * mk-sbuild: default to using UTC for schroots (LP: #2097159).
+  * syncpackage: s/syncblacklist/syncblocklist/g
+  * syncpackage: Cache the sync blocklist in-memory, so it's not fetched
+    multiple times when syncing more than one package.
+  * syncpackage: Catch exceptions cleanly, simply skipping to the next
+    package (erring on the side of caution) if there is an error doing the
+    download (LP: #1943286).
+
+ -- Simon Quigley <tsimonq2@debian.org>  Tue, 04 Mar 2025 13:43:15 -0600
+
+ubuntu-dev-tools (0.205) unstable; urgency=medium
+
+  * [syncpackage] When syncing multiple packages, if one of the packages is in
+    the sync blocklist, do not exit, simply continue.
+  * [syncpackage] Do not use exit(1) on an error or exception unless it
+    applies to all packages, instead return None so we can continue to the
+    next package.
+  * [syncpackage] Add support for -y or --yes, noted that it should be used
+    with care.
+  * Update Standards-Version to 4.7.2, no changes needed.
+
+ -- Simon Quigley <tsimonq2@debian.org>  Sat, 01 Mar 2025 11:29:54 -0600
+
 ubuntu-dev-tools (0.204) unstable; urgency=medium
 
   [ Simon Quigley ]

debian/control

@@ -31,7 +31,7 @@ Build-Depends:
  python3-requests <!nocheck>,
  python3-setuptools,
  python3-yaml <!nocheck>,
-Standards-Version: 4.7.1
+Standards-Version: 4.7.2
 Rules-Requires-Root: no
 Vcs-Git: https://git.launchpad.net/ubuntu-dev-tools
 Vcs-Browser: https://git.launchpad.net/ubuntu-dev-tools

doc/syncpackage.1

@@ -58,7 +58,7 @@ Display more progress information.
 \fB\-F\fR, \fB\-\-fakesync\fR
 Perform a fakesync, to work around a tarball mismatch between Debian and
 Ubuntu.
-This option ignores blacklisting, and performs a local sync.
+This option ignores blocklisting, and performs a local sync.
 It implies \fB\-\-no\-lp\fR, and will leave a signed \fB.changes\fR file
 for you to upload.
 .TP

import-bug-from-debian

@@ -150,7 +150,7 @@ def process_bugs(
 
     err = False
     for bug in bugs:
-        ubupackage = package = bug.source
+        ubupackage = bug.source
         if package:
             ubupackage = package
         bug_num = bug.bug_num

mk-sbuild

@@ -155,6 +155,7 @@ proxy="_unset_"
 DEBOOTSTRAP_NO_CHECK_GPG=0
 EATMYDATA=1
 CCACHE=0
+USE_PKGBINARYMANGLER=0
 
 while :; do
     case "$1" in
@@ -667,6 +668,7 @@ ubuntu)
     if ubuntu_dist_ge "$RELEASE" "edgy"; then
         # Add pkgbinarymangler (edgy and later)
         BUILD_PKGS="$BUILD_PKGS pkgbinarymangler"
+        USE_PKGBINARYMANGLER=1
         # Disable recommends for a smaller chroot (gutsy and later only)
         if ubuntu_dist_ge "$RELEASE" "gutsy"; then
             BUILD_PKGS="--no-install-recommends $BUILD_PKGS"
@@ -926,8 +928,8 @@ if [ -n "$TEMP_PREFERENCES" ]; then
     sudo mv "$TEMP_PREFERENCES" $MNT/etc/apt/preferences.d/proposed.pref
 fi
 
-# Copy the timezone (comment this out if you want to leave the chroot at UTC)
-sudo cp -P --remove-destination /etc/localtime /etc/timezone "$MNT"/etc/
+# Copy the timezone (uncomment this if you want to use your local time zone)
+#sudo cp -P --remove-destination /etc/localtime /etc/timezone "$MNT"/etc/
 # Create a schroot entry for this chroot
 TEMP_SCHROOTCONF=`mktemp -t schrootconf-XXXXXX`
 TEMPLATE_SCHROOTCONF=~/.mk-sbuild.schroot.conf
@@ -1046,6 +1048,25 @@ EOF
 EOM
 fi
 
+if [ "$USE_PKGBINARYMANGLER" = 1 ]; then
+    sudo bash -c "cat >> $MNT/finish.sh" <<EOM
+   mkdir -p /etc/pkgbinarymangler/
+   cat > /etc/pkgbinarymangler/maintainermangler.conf <<EOF
+# pkgmaintainermangler configuration file
+
+# pkgmaintainermangler will do nothing unless enable is set to "true"
+enable: true
+
+# Configure what happens if /CurrentlyBuilding is present, but invalid
+# (i. e. it does not contain a Package: field). If "ignore" (default),
+# the file is ignored (i. e. the Maintainer field is mangled) and a
+# warning is printed. If "fail" (or any other value), pkgmaintainermangler
+# exits with an error, which causes a package build to fail.
+invalid_currentlybuilding: ignore
+EOF
+EOM
+fi
+
 if [ -n "$TARGET_ARCH" ]; then
     sudo bash -c "cat >> $MNT/finish.sh" <<EOM
 # Configure target architecture
@@ -1064,7 +1085,7 @@ apt-get update || true
 echo set debconf/frontend Noninteractive | debconf-communicate
 echo set debconf/priority critical | debconf-communicate
 # Install basic build tool set, trying to match buildd
-apt-get -y --force-yes install $BUILD_PKGS
+apt-get -y --force-yes -o Dpkg::Options::="--force-confold" install $BUILD_PKGS
 # Set up expected /dev entries
 if [ ! -r /dev/stdin ];  then ln -s /proc/self/fd/0 /dev/stdin;  fi
 if [ ! -r /dev/stdout ]; then ln -s /proc/self/fd/1 /dev/stdout; fi

syncpackage

@@ -49,6 +49,7 @@ from ubuntutools.requestsync.mail import get_debian_srcpkg as requestsync_mail_g
 from ubuntutools.version import Version
 
 Logger = getLogger()
+cached_sync_blocklist = None
 
 
 def remove_signature(dscname):
@@ -143,7 +144,7 @@ def sync_dsc(
     if ubuntu_ver.is_modified_in_ubuntu():
         if not force:
             Logger.error("--force is required to discard Ubuntu changes.")
-            sys.exit(1)
+            return None
 
         Logger.warning(
             "Overwriting modified Ubuntu version %s, setting current version to %s",
@@ -157,7 +158,7 @@ def sync_dsc(
         src_pkg.pull()
     except DownloadError as e:
         Logger.error("Failed to download: %s", str(e))
-        sys.exit(1)
+        return None
     src_pkg.unpack()
 
     needs_fakesync = not (need_orig or ubu_pkg.verify_orig())
@@ -166,13 +167,13 @@ def sync_dsc(
         Logger.warning("Performing a fakesync")
     elif not needs_fakesync and fakesync:
         Logger.error("Fakesync not required, aborting.")
-        sys.exit(1)
+        return None
     elif needs_fakesync and not fakesync:
         Logger.error(
             "The checksums of the Debian and Ubuntu packages "
             "mismatch. A fake sync using --fakesync is required."
         )
-        sys.exit(1)
+        return None
 
     if fakesync:
         # Download Ubuntu files (override Debian source tarballs)
@@ -180,7 +181,7 @@ def sync_dsc(
             ubu_pkg.pull()
         except DownloadError as e:
             Logger.error("Failed to download: %s", str(e))
-            sys.exit(1)
+            return None
 
     # change into package directory
     directory = src_pkg.source + "-" + new_ver.upstream_version
@@ -265,7 +266,7 @@ def sync_dsc(
         returncode = subprocess.call(cmd)
         if returncode != 0:
             Logger.error("Source-only build with debuild failed. Please check build log above.")
-            sys.exit(1)
+            return None
 
 
 def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror):
@@ -295,7 +296,7 @@ def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror):
             udtexceptions.SeriesNotFoundException,
         ) as e:
             Logger.error(str(e))
-            sys.exit(1)
+            return None
         if version is None:
             version = Version(debian_srcpkg.getVersion())
         try:
@@ -306,7 +307,7 @@ def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror):
             ubuntu_version = Version("~")
         except udtexceptions.SeriesNotFoundException as e:
             Logger.error(str(e))
-            sys.exit(1)
+            return None
         if ubuntu_version >= version:
             # The LP importer is maybe out of date
             debian_srcpkg = requestsync_mail_get_debian_srcpkg(package, dist)
@@ -320,7 +321,7 @@ def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror):
                     ubuntu_version,
                     ubuntu_release,
                 )
-                sys.exit(1)
+                return None
         if component is None:
             component = debian_srcpkg.getComponent()
 
@@ -329,7 +330,7 @@ def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror):
     return DebianSourcePackage(package, version.full_version, component, mirrors=mirrors)
 
 
-def copy(src_pkg, release, bugs, sponsoree=None, simulate=False, force=False):
+def copy(src_pkg, release, bugs, sponsoree=None, simulate=False, force=False, yes=False):
     """Copy a source package from Debian to Ubuntu using the Launchpad API."""
     ubuntu = Distribution("ubuntu")
     debian_archive = Distribution("debian").getArchive()
@@ -352,7 +353,7 @@ def copy(src_pkg, release, bugs, sponsoree=None, simulate=False, force=False):
             "Debian version %s has not been picked up by LP yet. Please try again later.",
             src_pkg.version,
         )
-        sys.exit(1)
+        return None
 
     try:
         ubuntu_spph = get_ubuntu_srcpkg(src_pkg.source, ubuntu_series, ubuntu_pocket)
@@ -373,7 +374,7 @@ def copy(src_pkg, release, bugs, sponsoree=None, simulate=False, force=False):
         base_version = ubuntu_version.get_related_debian_version()
         if not force and ubuntu_version.is_modified_in_ubuntu():
             Logger.error("--force is required to discard Ubuntu changes.")
-            sys.exit(1)
+            return None
 
         # Check whether a fakesync would be required.
         if not src_pkg.dsc.compare_dsc(ubuntu_pkg.dsc):
@@ -381,7 +382,7 @@ def copy(src_pkg, release, bugs, sponsoree=None, simulate=False, force=False):
                 "The checksums of the Debian and Ubuntu packages "
                 "mismatch. A fake sync using --fakesync is required."
             )
-            sys.exit(1)
+            return None
     except udtexceptions.PackageNotFoundException:
         base_version = Version("~")
         Logger.info(
@@ -402,9 +403,10 @@ def copy(src_pkg, release, bugs, sponsoree=None, simulate=False, force=False):
 
     if sponsoree:
         Logger.info("Sponsoring this sync for %s (%s)", sponsoree.display_name, sponsoree.name)
-    answer = YesNoQuestion().ask("Sync this package", "no")
-    if answer != "yes":
-        return
+    if not yes:
+        answer = YesNoQuestion().ask("Sync this package", "no")
+        if answer != "yes":
+            return
 
     try:
         ubuntu_archive.copyPackage(
@@ -419,26 +421,29 @@ def copy(src_pkg, release, bugs, sponsoree=None, simulate=False, force=False):
     except HTTPError as error:
         Logger.error("HTTP Error %s: %s", error.response.status, error.response.reason)
         Logger.error(error.content)
-        sys.exit(1)
+        return None
 
     Logger.info("Request succeeded; you should get an e-mail once it is processed.")
     bugs = sorted(set(bugs))
     if bugs:
         Logger.info("Launchpad bugs to be closed: %s", ", ".join(str(bug) for bug in bugs))
         Logger.info("Please wait for the sync to be successful before closing bugs.")
-        answer = YesNoQuestion().ask("Close bugs", "yes")
-        if answer == "yes":
+        if yes:
             close_bugs(bugs, src_pkg.source, src_pkg.version.full_version, changes, sponsoree)
+        else:
+            answer = YesNoQuestion().ask("Close bugs", "yes")
+            if answer == "yes":
+                close_bugs(bugs, src_pkg.source, src_pkg.version.full_version, changes, sponsoree)
 
 
-def is_blacklisted(query):
-    """Determine if package "query" is in the sync blacklist
-    Returns tuple of (blacklisted, comments)
-    blacklisted is one of False, 'CURRENT', 'ALWAYS'
+def is_blocklisted(query):
+    """Determine if package "query" is in the sync blocklist
+    Returns tuple of (blocklisted, comments)
+    blocklisted is one of False, 'CURRENT', 'ALWAYS'
     """
     series = Launchpad.distributions["ubuntu"].current_series
     lp_comments = series.getDifferenceComments(source_package_name=query)
-    blacklisted = False
+    blocklisted = False
     comments = [
         f"{c.body_text}\n  -- {c.comment_author.name}"
         f"  {c.comment_date.strftime('%a, %d %b %Y %H:%M:%S +0000')}"
@@ -446,32 +451,38 @@ def is_blacklisted(query):
     ]
 
     for diff in series.getDifferencesTo(source_package_name_filter=query):
-        if diff.status == "Blacklisted current version" and blacklisted != "ALWAYS":
-            blacklisted = "CURRENT"
+        if diff.status == "Blacklisted current version" and blocklisted != "ALWAYS":
+            blocklisted = "CURRENT"
         if diff.status == "Blacklisted always":
-            blacklisted = "ALWAYS"
+            blocklisted = "ALWAYS"
 
-    # Old blacklist:
-    url = "https://ubuntu-archive-team.ubuntu.com/sync-blacklist.txt"
-    with urllib.request.urlopen(url) as f:
-        applicable_lines = []
-        for line in f:
-            line = line.decode("utf-8")
-            if not line.strip():
-                applicable_lines = []
-                continue
-            applicable_lines.append(line)
-            try:
-                line = line[: line.index("#")]
-            except ValueError:
-                pass
-            source = line.strip()
-            if source and fnmatch.fnmatch(query, source):
-                comments += ["From sync-blacklist.txt:"] + applicable_lines
-                blacklisted = "ALWAYS"
-                break
+    global cached_sync_blocklist
+    if not cached_sync_blocklist:
+        url = "https://ubuntu-archive-team.ubuntu.com/sync-blocklist.txt"
+        try:
+            with urllib.request.urlopen(url) as f:
+                cached_sync_blocklist = f.read().decode("utf-8")
+        except:
+            print("WARNING: unable to download the sync blocklist. Erring on the side of caution.")
+            return ("ALWAYS", "INTERNAL ERROR: Unable to fetch sync blocklist")
 
-    return (blacklisted, comments)
+    applicable_lines = []
+    for line in cached_sync_blocklist.splitlines():
+        if not line.strip():
+            applicable_lines = []
+            continue
+        applicable_lines.append(line)
+        try:
+            line = line[:line.index("#")]
+        except ValueError:
+            pass
+        source = line.strip()
+        if source and fnmatch.fnmatch(query, source):
+            comments += ["From sync-blocklist.txt:"] + applicable_lines
+            blocklisted = "ALWAYS"
+            break
+
+    return (blocklisted, comments)
 
 
 def close_bugs(bugs, package, version, changes, sponsoree):
@@ -508,6 +519,12 @@ def parse():
     epilog = f"See {os.path.basename(sys.argv[0])}(1) for more info."
     parser = argparse.ArgumentParser(usage=usage, epilog=epilog)
 
+    parser.add_argument(
+        "-y",
+        "--yes",
+        action="store_true",
+        help="Automatically sync without prompting. Use with caution and care."
+    )
     parser.add_argument("-d", "--distribution", help="Debian distribution to sync from.")
     parser.add_argument("-r", "--release", help="Specify target Ubuntu release.")
     parser.add_argument("-V", "--debian-version", help="Specify the version to sync from.")
@@ -712,36 +729,38 @@ def main():
             args.release,
             args.debian_mirror,
         )
+        if not src_pkg:
+            continue
 
-        blacklisted, comments = is_blacklisted(src_pkg.source)
-        blacklist_fail = False
-        if blacklisted:
+        blocklisted, comments = is_blocklisted(src_pkg.source)
+        blocklist_fail = False
+        if blocklisted:
             messages = []
 
-            if blacklisted == "CURRENT":
+            if blocklisted == "CURRENT":
                 Logger.debug(
-                    "Source package %s is temporarily blacklisted "
-                    "(blacklisted_current). "
+                    "Source package %s is temporarily blocklisted "
+                    "(blocklisted_current). "
                     "Ubuntu ignores these for now. "
                     "See also LP: #841372",
                     src_pkg.source,
                 )
             else:
                 if args.fakesync:
-                    messages += ["Doing a fakesync, overriding blacklist."]
+                    messages += ["Doing a fakesync, overriding blocklist."]
                 else:
-                    blacklist_fail = True
+                    blocklist_fail = True
                     messages += [
                         "If this package needs a fakesync, use --fakesync",
                         "If you think this package shouldn't be "
-                        "blacklisted, please file a bug explaining your "
+                        "blocklisted, please file a bug explaining your "
                         "reasoning and subscribe ~ubuntu-archive.",
                     ]
 
-            if blacklist_fail:
-                Logger.error("Source package %s is blacklisted.", src_pkg.source)
-            elif blacklisted == "ALWAYS":
-                Logger.info("Source package %s is blacklisted.", src_pkg.source)
+            if blocklist_fail:
+                Logger.error("Source package %s is blocklisted.", src_pkg.source)
+            elif blocklisted == "ALWAYS":
+                Logger.info("Source package %s is blocklisted.", src_pkg.source)
             if messages:
                 for message in messages:
                     for line in textwrap.wrap(message):
@@ -753,14 +772,15 @@ def main():
                 for line in textwrap.wrap(comment):
                     Logger.info("  %s", line)
 
-        if blacklist_fail:
-            sys.exit(1)
+        if blocklist_fail:
+            continue
 
         if args.lp:
-            copy(src_pkg, args.release, args.bugs, sponsoree, args.simulate, args.force)
+            if not copy(src_pkg, args.release, args.bugs, sponsoree, args.simulate, args.force, args.yes):
+                continue
         else:
             os.environ["DEB_VENDOR"] = "Ubuntu"
-            sync_dsc(
+            if not sync_dsc(
                 src_pkg,
                 args.distribution,
                 args.release,
@@ -772,7 +792,8 @@ def main():
                 args.simulate,
                 args.force,
                 args.fakesync,
-            )
+            ):
+                continue
 
 
 if __name__ == "__main__":

ubuntutools/archive.py

@@ -165,6 +165,7 @@ class SourcePackage(ABC):
         series = kwargs.get("series")
         pocket = kwargs.get("pocket")
         status = kwargs.get("status")
+        arch = kwargs.get("arch")
         verify_signature = kwargs.get("verify_signature", False)
         try_binary = kwargs.get("try_binary", True)
 
@@ -184,6 +185,7 @@ class SourcePackage(ABC):
         self._series = series
         self._pocket = pocket
         self._status = status
+        self._arch = arch
         # dscfile can be either a path or an URL.  misc.py's download() will
         # later fiture it out
         self._dsc_source = dscfile
@@ -252,6 +254,7 @@ class SourcePackage(ABC):
             )
 
             try:
+                params["archtag"] = self._arch
                 bpph = archive.getBinaryPackage(self.source, **params)
             except PackageNotFoundException as bpnfe:
                 # log binary lookup failure, in case it provides hints

ubuntutools/config.py

@@ -68,21 +68,19 @@ class UDTConfig:
         config = {}
         for filename in ("/etc/devscripts.conf", "~/.devscripts"):
             try:
-                f = open(os.path.expanduser(filename), "r", encoding="utf-8")
+                with open(os.path.expanduser(filename), "r", encoding="utf-8") as f:
+                    content = f.read()
             except IOError:
                 continue
-            for line in f:
-                parsed = shlex.split(line, comments=True)
-                if len(parsed) > 1:
-                    Logger.warning(
-                        "Cannot parse variable assignment in %s: %s",
-                        getattr(f, "name", "<config>"),
-                        line,
-                    )
-                if len(parsed) >= 1 and "=" in parsed[0]:
-                    key, value = parsed[0].split("=", 1)
+            try:
+                tokens = shlex.split(content, comments=True)
+            except ValueError as e:
+                Logger.error("Error parsing %s: %s", filename, e)
+                continue
+            for token in tokens:
+                if "=" in token:
+                    key, value = token.split("=", 1)
                     config[key] = value
-            f.close()
         return config
 
     def get_value(self, key, default=None, boolean=False, compat_keys=()):

ubuntutools/pullpkg.py

@@ -340,6 +340,7 @@ class PullPkg:
 
         params = {}
         params["package"] = options["package"]
+        params["arch"] = options["arch"]
 
         if options["release"]:
             (release, version, pocket) = self.parse_release_and_version(