ubuntu-dev-tools/backportpackage

#!/usr/bin/python3
# -*- coding: utf-8 -*-
# ##################################################################
#
# Copyright (C) 2010-2011, Evan Broder <evan@ebroder.net>
# Copyright (C) 2010, Benjamin Drung <bdrung@ubuntu.com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# See file /usr/share/common-licenses/GPL-2 for more details.
#
# ##################################################################

import argparse
import glob
import os
import shutil
import subprocess
import sys
import tempfile
from urllib.parse import quote

try:
    import lsb_release
except ImportError:
    lsb_release = None
from distro_info import DebianDistroInfo, UbuntuDistroInfo
from httplib2 import Http, HttpLib2Error

from ubuntutools import getLogger
from ubuntutools.archive import DebianSourcePackage, DownloadError, UbuntuSourcePackage
from ubuntutools.builder import get_builder
from ubuntutools.config import UDTConfig, ubu_email
from ubuntutools.lp.lpapicache import (
    Distribution,
    Launchpad,
    PackageNotFoundException,
    SeriesNotFoundException,
)
from ubuntutools.misc import codename_to_distribution, system_distribution, vendor_to_distroinfo
from ubuntutools.question import YesNoQuestion

Logger = getLogger()


def error(msg, *args):
    Logger.error(msg, *args)
    sys.exit(1)


def check_call(cmd, *args, **kwargs):
    Logger.debug(" ".join(cmd))
    ret = subprocess.call(cmd, *args, **kwargs)
    if ret != 0:
        error("%s returned %d.", cmd[0], ret)


def parse(argv):
    usage = "%(prog)s [options] <source package name or .dsc URL/file>"
    parser = argparse.ArgumentParser(usage=usage)
    parser.add_argument(
        "-d",
        "--destination",
        metavar="DEST",
        dest="dest_releases",
        default=[],
        action="append",
        help="Backport to DEST release (default: current release)",
    )
    parser.add_argument(
        "-s",
        "--source",
        metavar="SOURCE",
        dest="source_release",
        help="Backport from SOURCE release (default: devel release)",
    )
    parser.add_argument(
        "-S",
        "--suffix",
        metavar="SUFFIX",
        help="Suffix to append to version number (default: ~ppa1 when uploading to a PPA)",
    )
    parser.add_argument(
        "-e",
        "--message",
        metavar="MESSAGE",
        default="No-change",
        help='Changelog message to use instead of "No-change" '
        "(default: No-change backport to DEST.)",
    )
    parser.add_argument(
        "-b",
        "--build",
        default=False,
        action="store_true",
        help="Build the package before uploading (default: %(default)s)",
    )
    parser.add_argument(
        "-B",
        "--builder",
        metavar="BUILDER",
        help="Specify the package builder (default: pbuilder)",
    )
    parser.add_argument(
        "-U",
        "--update",
        default=False,
        action="store_true",
        help="Update the build environment before attempting to build",
    )
    parser.add_argument("-u", "--upload", metavar="UPLOAD", help="Specify an upload destination")
    parser.add_argument(
        "-k", "--key", dest="keyid", help="Specify the key ID to be used for signing."
    )
    parser.add_argument(
        "--dont-sign", dest="keyid", action="store_false", help="Do not sign the upload."
    )
    parser.add_argument(
        "-y",
        "--yes",
        dest="prompt",
        default=True,
        action="store_false",
        help="Do not prompt before uploading to a PPA",
    )
    parser.add_argument(
        "-v", "--version", metavar="VERSION", help="Package version to backport (or verify)"
    )
    parser.add_argument(
        "-w",
        "--workdir",
        metavar="WORKDIR",
        help="Specify a working directory (default: temporary dir)",
    )
    parser.add_argument(
        "-r",
        "--release-pocket",
        default=False,
        action="store_true",
        help="Target the release pocket in the .changes file. "
        "Necessary (and default) for uploads to PPAs",
    )
    parser.add_argument(
        "-c", "--close", metavar="BUG", help="Bug to close in the changelog entry."
    )
    parser.add_argument(
        "-m", "--mirror", metavar="URL", help="Preferred mirror (default: Launchpad)"
    )
    parser.add_argument(
        "-l",
        "--lpinstance",
        metavar="INSTANCE",
        help="Launchpad instance to connect to (default: production)",
    )
    parser.add_argument(
        "--no-conf",
        default=False,
        action="store_true",
        help="Don't read config files or environment variables",
    )
    parser.add_argument("package_or_dsc", help=argparse.SUPPRESS)

    args = parser.parse_args(argv)
    config = UDTConfig(args.no_conf)
    if args.builder is None:
        args.builder = config.get_value("BUILDER")
    if not args.update:
        args.update = config.get_value("UPDATE_BUILDER", boolean=True)
    if args.workdir is None:
        args.workdir = config.get_value("WORKDIR")
    if args.lpinstance is None:
        args.lpinstance = config.get_value("LPINSTANCE")
    if args.upload is None:
        args.upload = config.get_value("UPLOAD")
    if args.keyid is None:
        args.keyid = config.get_value("KEYID")
    if not args.upload and not args.workdir:
        parser.error("Please specify either a working dir or an upload target!")
    if args.upload and args.upload.startswith("ppa:"):
        args.release_pocket = True

    return args, config


def find_release_package(mirror, workdir, package, version, source_release, config):
    srcpkg = None

    if source_release:
        distribution = codename_to_distribution(source_release)
        if not distribution:
            error("Unknown release codename %s", source_release)
        info = vendor_to_distroinfo(distribution)()
        source_release = info.codename(source_release, default=source_release)
    else:
        distribution = system_distribution()
    mirrors = [mirror] if mirror else []

    mirrors.append(config.get_value(f"{distribution.upper()}_MIRROR"))

    if not version:
        archive = Distribution(distribution.lower()).getArchive()
        try:
            spph = archive.getSourcePackage(package, source_release)
        except (SeriesNotFoundException, PackageNotFoundException) as e:
            error("%s", str(e))
        version = spph.getVersion()

    if distribution == "Debian":
        srcpkg = DebianSourcePackage(package, version, workdir=workdir, mirrors=mirrors)
    elif distribution == "Ubuntu":
        srcpkg = UbuntuSourcePackage(package, version, workdir=workdir, mirrors=mirrors)

    return srcpkg


def find_package(mirror, workdir, package, version, source_release, config):
    "Returns the SourcePackage"
    if package.endswith(".dsc"):
        # Here we are using UbuntuSourcePackage just because we don't have any
        # "general" class that is safely instantiable (as SourcePackage is an
        # abstract class).  None of the distribution-specific details within
        # UbuntuSourcePackage is relevant for this use case.
        return UbuntuSourcePackage(
            version=version, dscfile=package, workdir=workdir, mirrors=(mirror,)
        )

    if not source_release and not version:
        info = vendor_to_distroinfo(system_distribution())
        source_release = info().devel()

    srcpkg = find_release_package(mirror, workdir, package, version, source_release, config)
    if version and srcpkg.version != version:
        error(
            "Requested backport of version %s but version of %s in %s is %s",
            version,
            package,
            source_release,
            srcpkg.version,
        )

    return srcpkg


def get_backport_version(version, suffix, upload, release):
    distribution = codename_to_distribution(release)
    if not distribution:
        error("Unknown release codename %s", release)
    if distribution == "Debian":
        debian_distro_info = DebianDistroInfo()
        debian_codenames = debian_distro_info.supported()
        if release in debian_codenames:
            release_version = debian_distro_info.version(release)
            if not release_version:
                error("Can't find the release version for %s", release)
            backport_version = f"{version}~bpo{release_version}+1"
        else:
            error("%s is not a supported release (%s)", release, debian_codenames)
    elif distribution == "Ubuntu":
        series = Distribution(distribution.lower()).getSeries(name_or_version=release)

        backport_version = f"{version}~bpo{series.version}.1"
    else:
        error("Unknown distribution «%s» for release «%s»", distribution, release)
    if suffix is not None:
        backport_version += suffix
    elif upload and upload.startswith("ppa:"):
        backport_version += "~ppa1"
    return backport_version


def get_old_version(source, release):
    try:
        distribution = codename_to_distribution(release)
        archive = Distribution(distribution.lower()).getArchive()
        pkg = archive.getSourcePackage(
            source, release, ("Release", "Security", "Updates", "Proposed", "Backports")
        )
        return pkg.getVersion()
    except (SeriesNotFoundException, PackageNotFoundException):
        pass
    return None


def get_backport_dist(release, release_pocket):
    if release_pocket:
        return release
    return f"{release}-backports"


def do_build(workdir, dsc, release, builder, update):
    builder = get_builder(builder)
    if not builder:
        return None

    if update:
        if 0 != builder.update(release):
            sys.exit(1)

    # builder.build is going to chdir to buildresult:
    workdir = os.path.realpath(workdir)
    return builder.build(os.path.join(workdir, dsc), release, os.path.join(workdir, "buildresult"))


def do_upload(workdir, package, bp_version, changes, upload, prompt):
    print(f"Please check {package} {bp_version} in file://{workdir} carefully!")
    if prompt or upload == "ubuntu":
        question = f"Do you want to upload the package to {upload}"
        answer = YesNoQuestion().ask(question, "yes")
        if answer == "no":
            return

    check_call(["dput", upload, changes], cwd=workdir)


def orig_needed(upload, workdir, pkg):
    """Avoid a -sa if possible"""
    if not upload or not upload.startswith("ppa:"):
        return True
    ppa = upload.split(":", 1)[1]
    user, ppa = ppa.split("/", 1)

    version = pkg.version.upstream_version

    http = Http()
    for filename in glob.glob(os.path.join(workdir, f"{pkg.source}_{version}.orig*")):
        url = (
            f"https://launchpad.net/~{quote(user)}/+archive/{quote(ppa)}/+sourcefiles"
            f"/{quote(pkg.source)}/{quote(pkg.version.full_version)}"
            f"/{quote(os.path.basename(filename))}"
        )
        try:
            headers = http.request(url, "HEAD")[0]
            if headers.status != 200 or not headers["content-location"].startswith(
                "https://launchpadlibrarian.net"
            ):
                return True
        except HttpLib2Error as e:
            Logger.debug(e)
            return True
    return False


def do_backport(
    workdir,
    pkg,
    suffix,
    message,
    close,
    release,
    release_pocket,
    build,
    builder,
    update,
    upload,
    keyid,
    prompt,
):
    dirname = f"{pkg.source}-{release}"
    srcdir = os.path.join(workdir, dirname)

    if os.path.exists(srcdir):
        question = f"Working directory {srcdir} already exists. Delete it?"
        if YesNoQuestion().ask(question, "no") == "no":
            sys.exit(1)
        shutil.rmtree(srcdir)

    pkg.unpack(dirname)

    bp_version = get_backport_version(pkg.version.full_version, suffix, upload, release)
    old_version = get_old_version(pkg.source, release)
    bp_dist = get_backport_dist(release, release_pocket)

    changelog = f"{message} backport to {release}."
    if close:
        changelog += f" (LP: #{close})"
    check_call(
        [
            "dch",
            "--force-bad-version",
            "--force-distribution",
            "--preserve",
            "--newversion",
            bp_version,
            "--distribution",
            bp_dist,
            changelog,
        ],
        cwd=srcdir,
    )

    cmd = ["debuild", "--no-lintian", "-S", "-nc", "-uc", "-us"]
    if orig_needed(upload, workdir, pkg):
        cmd.append("-sa")
    else:
        cmd.append("-sd")
    if old_version:
        cmd.append(f"-v{old_version}")
    env = os.environ.copy()
    # An ubuntu.com e-mail address would make dpkg-buildpackage fail if there
    # wasn't an Ubuntu maintainer for an ubuntu-versioned package. LP: #1007042
    env.pop("DEBEMAIL", None)
    check_call(cmd, cwd=srcdir, env=env)

    fn_base = pkg.source + "_" + bp_version.split(":", 1)[-1]
    changes = fn_base + "_source.changes"

    if build:
        if 0 != do_build(workdir, fn_base + ".dsc", release, builder, update):
            sys.exit(1)

    # None: sign with the default signature. False: don't sign
    if keyid is not False:
        cmd = ["debsign"]
        if keyid:
            cmd.append("-k" + keyid)
        cmd.append(changes)
        check_call(cmd, cwd=workdir)
    if upload:
        do_upload(workdir, pkg.source, bp_version, changes, upload, prompt)

    shutil.rmtree(srcdir)


def main(argv):
    ubu_email()

    args, config = parse(argv[1:])

    Launchpad.login_anonymously(service=args.lpinstance)

    if not args.dest_releases:
        if lsb_release:
            distinfo = lsb_release.get_distro_information()
            try:
                current_distro = distinfo["ID"]
            except KeyError:
                error("No destination release specified and unable to guess yours.")
        else:
            err, current_distro = subprocess.getstatusoutput("lsb_release --id --short")
            if err:
                error("Could not run lsb_release to retrieve distribution")

        if current_distro == "Ubuntu":
            args.dest_releases = [UbuntuDistroInfo().lts()]
        elif current_distro == "Debian":
            args.dest_releases = [DebianDistroInfo().stable()]
        else:
            error("Unknown distribution %s, can't guess target release", current_distro)

    if args.workdir:
        workdir = os.path.expanduser(args.workdir)
    else:
        workdir = tempfile.mkdtemp(prefix="backportpackage-")

    if not os.path.exists(workdir):
        os.makedirs(workdir)

    try:
        pkg = find_package(
            args.mirror, workdir, args.package_or_dsc, args.version, args.source_release, config
        )
        pkg.pull()

        for release in args.dest_releases:
            do_backport(
                workdir,
                pkg,
                args.suffix,
                args.message,
                args.close,
                release,
                args.release_pocket,
                args.build,
                args.builder,
                args.update,
                args.upload,
                args.keyid,
                args.prompt,
            )
    except DownloadError as e:
        error("%s", str(e))
    finally:
        if not args.workdir:
            shutil.rmtree(workdir)


if __name__ == "__main__":
    sys.exit(main(sys.argv))