ci-tools/pending-packages

#!/usr/bin/env python3
#
# Copyright (C) 2024 Simon Quigley <tsimonq2@ubuntu.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

import argparse
from datetime import datetime, timedelta, timezone
from launchpadlib.launchpad import Launchpad

print(f"Logging into Launchpad...")
launchpad = Launchpad.login_with("pending-packages", "production", version="devel")

print("Logged in. Initializing repositories...")
ubuntu = launchpad.distributions["ubuntu"]
lubuntu_ci = launchpad.people["lubuntu-ci"]
regular = lubuntu_ci.getPPAByName(distribution=ubuntu, name="unstable-ci")
proposed = lubuntu_ci.getPPAByName(distribution=ubuntu, name="unstable-ci-proposed")

parser = argparse.ArgumentParser()
parser.add_argument("release")
args = parser.parse_args()
series = ubuntu.getSeries(name_or_version=args.release)

# First, check if any sources are still publishing
print("Repositories initialized. Checking for pending sources...")
records = [regular.getPublishedSources(status="Pending", distro_series=series),
           proposed.getPublishedSources(status="Pending", distro_series=series)]
total_pending = sum([len(i) for i in records])
has_pending = total_pending != 0

if has_pending:
    print(f"Total sources pending: {total_pending}")
    print("Sources are still pending, not running Britney")
    exit(1)

# Finally, check if any builds are still running/queued
print("No pending sources, continuing. Checking for pending builds...")
total_pending = 0
total_retried = 0
for archive in [proposed, regular]:
    one_hour_ago = datetime.now(timezone.utc) - timedelta(hours=1)
    for source in archive.getPublishedSources(status="Published", distro_series=series):
        for build in source.getBuilds():
            if build.buildstate == "Currently building":
                if build.date_started >= one_hour_ago:
                    total_pending += 1
            elif build.buildstate == "Needs building":
                total_pending += 1
            # This isn't technically related, but retry failed builds without logs
            elif build.buildstate == "Chroot problem" or (build.buildstate == "Failed to build" and not build.build_log_url):
                if build.can_be_retried:
                    build.retry()
                    total_pending += 1
                    total_retried += 1

if total_retried != 0:
    print(f"Total builds retried due to builder flakiness: {total_retried}")

if total_pending != 0:
    print(f"Total builds pending: {total_pending}")
    print("Builds are still running, not running Britney")
    exit(1)

print("No pending builds, continuing. Checking for pending binaries...")
has_pending = False
for pocket in [proposed, regular]:
    if has_pending:
        break
    three_hours_ago = datetime.now(timezone.utc) - timedelta(hours=3)
    check_builds = set()
    current_builds = set()
    source_packages = []

    for build in pocket.getBuildRecords(build_state="Successfully built"):
        if build.datebuilt < three_hours_ago:
            del source_packages
            break
        check_builds.add(build.title)
        source_package = build.current_source_publication
        if source_package and source_package.distro_series == series and source_package not in source_packages:
            source_packages.append(source_package)
            for binary in source_package.getPublishedBinaries():
                current_builds.add(binary.build.title)

    has_pending = not check_builds.issuperset(current_builds) or has_pending

if has_pending:
    print("Binaries are still pending, not running Britney")
    exit(1)

print("All clear. Starting Britney.")