Port pending-packages to C++ using launchpadlib-cpp

cpp/fetch-indexes.cpp

@@ -15,6 +15,12 @@
 
 #include "utilities.h"
 
+#include "launchpad.h"
+#include "archive.h"
+#include "distribution.h"
+#include "person.h"
+#include "source.h"
+
 #include <iostream>
 #include <fstream>
 #include <string>
@@ -33,8 +39,9 @@
 #include <getopt.h>
 #include <yaml-cpp/yaml.h>
 #include <sstream>
-
-#include "utilities.h"
+#include <optional>
+#include <unordered_set>
+#include <set>
 
 namespace fs = std::filesystem;
 
@@ -44,6 +51,10 @@ void processRelease(const std::string& release, const YAML::Node& config);
 void refresh(const std::string& url, const std::string& pocket, const std::string& britneyCache, std::mutex& logMutex);
 int executeAndLog(const std::string& command);
 
+// Global Launchpad variable, initialized once for reuse
+static std::optional<launchpad> global_lp_opt;
+static launchpad* global_lp = nullptr;
+
 // Execute a command and stream its output to std::cout in real time
 int executeAndLog(const std::string& command) {
     std::string fullCommand = command + " 2>&1"; // Redirect stderr to stdout
@@ -56,7 +67,7 @@ int executeAndLog(const std::string& command) {
     char buffer[256];
     while (fgets(buffer, sizeof(buffer), pipe) != nullptr) {
         std::cout << buffer;
-        std::cout.flush(); // Ensure real-time logging
+        std::cout.flush();
     }
 
     int exitCode = pclose(pipe);
@@ -67,6 +78,194 @@ int executeAndLog(const std::string& command) {
     }
 }
 
+// This function replaces the Python "pending-packages" script logic.
+// Everything is done in one function, using the Launchpad C++ logic shown before.
+int check_pending_packages(const std::string& release) {
+    // Ensure global_lp is initialized once globally
+    if (!global_lp_opt.has_value()) {
+        global_lp_opt = launchpad::login();
+        if (!global_lp_opt.has_value()) {
+            std::cerr << "Failed to authenticate with Launchpad.\n";
+            return 1;
+        }
+        global_lp = &global_lp_opt.value();
+    }
+
+    auto lp = global_lp;
+
+    std::cout << "Logging into Launchpad..." << std::endl;
+    // We already logged in globally above.
+    std::cout << "Logged in. Initializing repositories..." << std::endl;
+
+    auto ubuntu_opt = lp->distributions["ubuntu"];
+    if (!ubuntu_opt.has_value()) {
+        std::cerr << "Failed to retrieve ubuntu.\n";
+        return 1;
+    }
+    distribution ubuntu = std::move(ubuntu_opt.value());
+
+    auto lubuntu_ci_opt = lp->people["lubuntu-ci"];
+    if (!lubuntu_ci_opt.has_value()) {
+        std::cerr << "Failed to retrieve lubuntu-ci.\n";
+        return 1;
+    }
+    person lubuntu_ci = std::move(lubuntu_ci_opt.value());
+
+    std::optional<archive> regular_opt = lubuntu_ci.getPPAByName(ubuntu, "unstable-ci");
+    if (!regular_opt.has_value()) {
+        std::cerr << "Failed to retrieve regular PPA.\n";
+        return 1;
+    }
+    archive regular = std::move(regular_opt.value());
+
+    std::optional<archive> proposed_opt = lubuntu_ci.getPPAByName(ubuntu, "unstable-ci-proposed");
+    if (!proposed_opt.has_value()) {
+        std::cerr << "Failed to retrieve proposed PPA.\n";
+        return 1;
+    }
+    archive proposed = std::move(proposed_opt.value());
+
+    // We need to get the distro series for "release"
+    auto series_opt = ubuntu.getSeries(release);
+    if (!series_opt.has_value()) {
+        std::cerr << "Failed to retrieve series for: " << release << std::endl;
+        return 1;
+    }
+    distro_series series = std::move(series_opt.value());
+
+    std::cout << "Repositories initialized. Checking for pending sources..." << std::endl;
+
+    // Check if any sources are pending in either regular or proposed
+    int total_pending = 0;
+    {
+        auto reg_pending = regular.getPublishedSources("Pending", series);
+        auto prop_pending = proposed.getPublishedSources("Pending", series);
+        total_pending = (int)reg_pending.size() + (int)prop_pending.size();
+    }
+
+    bool has_pending = (total_pending != 0);
+    if (has_pending) {
+        std::cout << "Total sources pending: " << total_pending << std::endl;
+        std::cout << "Sources are still pending, not running Britney" << std::endl;
+        return 1;
+    }
+
+    std::cout << "No pending sources, continuing. Checking for pending builds..." << std::endl;
+    total_pending = 0;
+    int total_retried = 0;
+    {
+        using namespace std::chrono;
+        auto now_utc = std::chrono::system_clock::now();
+        auto one_hour_ago = now_utc - std::chrono::hours(1);
+
+        // Convert one_hour_ago to a time_point that matches the build date time usage
+        // Our build.date_started presumably returns a std::chrono::system_clock::time_point
+        auto archives = {proposed, regular};
+        for (auto& archv : archives) {
+            for (auto src : archv.getPublishedSources("Published", series)) {
+                for (auto build : src.getBuilds()) {
+                    auto bs = build.buildstate;
+                    if (bs == "Currently building") {
+                        if (build.date_started.has_value() && build.date_started.value() >= one_hour_ago) {
+                            total_pending += 1;
+                        }
+                    } else if (bs == "Needs building") {
+                        total_pending += 1;
+                    } else if (bs == "Chroot problem" ||
+                               (bs == "Failed to build" && build.build_log_url.empty())) {
+                        // Retry failed builds without logs
+                        if (build.can_be_retried) {
+                            if (build.retry()) {
+                                total_pending += 1;
+                                total_retried += 1;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    if (total_retried != 0) {
+        std::cout << "Total builds retried due to builder flakiness: " << total_retried << std::endl;
+    }
+
+    if (total_pending != 0) {
+        std::cout << "Total builds pending: " << total_pending << std::endl;
+        std::cout << "Builds are still running, not running Britney" << std::endl;
+        return 1;
+    }
+
+    std::cout << "No pending builds, continuing. Checking for pending binaries..." << std::endl;
+
+    // Check if binaries are pending
+    // This logic replicates the python code:
+    // For each of [proposed, regular], if binaries are not all published after a certain grace period,
+    // we consider them pending.
+    has_pending = false;
+
+    {
+        auto archives = {proposed, regular};
+        for (auto& pocket : archives) {
+            if (has_pending) break;
+
+            auto three_hours_ago = std::chrono::system_clock::now() - std::chrono::hours(3);
+            std::set<std::string> check_builds;
+            std::set<std::string> current_builds;
+            std::vector<source> source_packages;
+
+            // Get successfully built records
+            for (auto build_record : pocket.getBuildRecords("Successfully built")) {
+                if (build_record.datebuilt.has_value() && build_record.datebuilt.value() < three_hours_ago) {
+                    // If any build is older than 3 hours, the python code breaks and doesn't check
+                    // further. We'll do the same.
+                    source_packages.clear();
+                    break;
+                }
+                check_builds.insert(build_record.title);
+                auto src_pub = build_record.current_source_publication;
+                if (src_pub.has_value() && src_pub.value().distro_series.name_or_version == series.name_or_version) {
+                    bool found = false;
+                    for (auto& sp : source_packages) {
+                        if (sp.self_link == src_pub.value().self_link) {
+                            found = true;
+                            break;
+                        }
+                    }
+                    if (!found && src_pub.has_value()) {
+                        source_packages.push_back(src_pub.value());
+                    }
+                }
+            }
+
+            // For each source package, get their published binaries and see if they're all in check_builds
+            for (auto& s : source_packages) {
+                for (auto bin : s.getPublishedBinaries()) {
+                    current_builds.insert(bin.build.title);
+                }
+            }
+
+            // If check_builds does not fully cover current_builds, we have pending binaries
+            for (auto& cb : current_builds) {
+                if (check_builds.find(cb) == check_builds.end()) {
+                    has_pending = true;
+                    break;
+                }
+            }
+        }
+    }
+
+    if (has_pending) {
+        std::cout << "Binaries are still pending, not running Britney" << std::endl;
+        return 1;
+    }
+
+    std::cout << "All clear. Starting Britney." << std::endl;
+
+    return 0;
+}
+
+
 int main(int argc, char* argv[]) {
     std::string configFilePath = "config.yaml";
 
@@ -110,7 +309,7 @@ int main(int argc, char* argv[]) {
     YAML::Node config;
     try {
         config = YAML::LoadFile(configFilePath);
-    } catch (const YAML::BadFile& e) {
+    } catch (const YAML::BadFile&) {
         std::cerr << "Error: Unable to open config file: " << configFilePath << std::endl;
         return 1;
     } catch (const YAML::ParserException& e) {
@@ -200,20 +399,10 @@ void processRelease(const std::string& RELEASE, const YAML::Node& config) {
     std::string SOURCE_PPA_URL = "https://ppa.launchpadcontent.net/" + LP_TEAM + "/" + SOURCE_PPA + "/ubuntu/dists/" + RELEASE + "/main";
     std::string DEST_PPA_URL = "https://ppa.launchpadcontent.net/" + LP_TEAM + "/" + DEST_PPA + "/ubuntu/dists/" + RELEASE + "/main";
 
-    // Get current timestamp
-    std::time_t now_c = std::time(nullptr);
-    char timestamp[20];
-    std::strftime(timestamp, sizeof(timestamp), "%Y%m%dT%H%M%S", std::gmtime(&now_c));
-    std::string BRITNEY_TIMESTAMP(timestamp);
-
-    std::cout << "Release: " << RELEASE << std::endl;
-    std::cout << "Timestamp: " << BRITNEY_TIMESTAMP << std::endl;
-
-    // Execute pending-packages script and capture its output
-    std::string pendingCmd = "./pending-packages " + RELEASE;
-    int pendingResult = executeAndLog(pendingCmd);
+    // Instead of calling pending-packages script, we call check_pending_packages function
+    int pendingResult = check_pending_packages(RELEASE);
     if (pendingResult != 0) {
-        std::cerr << "Error: pending-packages script failed for release " << RELEASE << std::endl;
+        std::cerr << "Error: pending-packages (now integrated check) failed for release " << RELEASE << std::endl;
         return;
     }
 
@@ -222,11 +411,9 @@ void processRelease(const std::string& RELEASE, const YAML::Node& config) {
     std::vector<std::thread> threads;
     std::mutex logMutex;
 
-    // Refresh package indexes
     std::vector<std::string> pockets = {RELEASE, RELEASE + "-updates"};
     std::vector<std::string> components = {"main", "restricted", "universe", "multiverse"};
 
-    // Loop over pockets, components, architectures to refresh package indexes
     for (const auto& pocket : pockets) {
         for (const auto& component : components) {
             for (const auto& arch : ARCHES) {
@@ -242,42 +429,45 @@ void processRelease(const std::string& RELEASE, const YAML::Node& config) {
         }
     }
 
-    // Treat the destination PPA as just another pocket
-    std::string pocket = RELEASE + "-ppa-proposed";
-    for (const auto& arch : ARCHES) {
-        std::string url = DEST_PPA_URL + "/binary-" + arch + "/Packages.gz";
-        threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
-    }
-    for (const auto& arch : PORTS_ARCHES) {
-        std::string url = DEST_PPA_URL + "/binary-" + arch + "/Packages.gz";
-        threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
-    }
     {
-        std::string url = DEST_PPA_URL + "/source/Sources.gz";
-        threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
+        // Treat DEST_PPA as proposed pocket
+        std::string pocket = RELEASE + "-ppa-proposed";
+        for (const auto& arch : ARCHES) {
+            std::string url = DEST_PPA_URL + "/binary-" + arch + "/Packages.gz";
+            threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
+        }
+        for (const auto& arch : PORTS_ARCHES) {
+            std::string url = DEST_PPA_URL + "/binary-" + arch + "/Packages.gz";
+            threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
+        }
+        {
+            std::string url = DEST_PPA_URL + "/source/Sources.gz";
+            threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
+        }
     }
 
-    // Get the source PPA
-    pocket = SOURCE_PPA + "-" + RELEASE;
-    for (const auto& arch : ARCHES) {
-        std::string url = SOURCE_PPA_URL + "/binary-" + arch + "/Packages.gz";
-        threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
-    }
-    for (const auto& arch : PORTS_ARCHES) {
-        std::string url = SOURCE_PPA_URL + "/binary-" + arch + "/Packages.gz";
-        threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
-    }
     {
-        std::string url = SOURCE_PPA_URL + "/source/Sources.gz";
-        threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
+        // SOURCE_PPA as unstable pocket
+        std::string pocket = SOURCE_PPA + "-" + RELEASE;
+        for (const auto& arch : ARCHES) {
+            std::string url = SOURCE_PPA_URL + "/binary-" + arch + "/Packages.gz";
+            threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
+        }
+        for (const auto& arch : PORTS_ARCHES) {
+            std::string url = SOURCE_PPA_URL + "/binary-" + arch + "/Packages.gz";
+            threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
+        }
+        {
+            std::string url = SOURCE_PPA_URL + "/source/Sources.gz";
+            threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
+        }
     }
 
-    // Wait for all threads to finish
     for (auto& th : threads) {
         th.join();
     }
 
-    // Process logs and delete them
+    // Process logs
     pid_t pid = getpid();
     std::string logPattern = std::to_string(pid) + "-wget-log";
 
@@ -285,7 +475,6 @@ void processRelease(const std::string& RELEASE, const YAML::Node& config) {
         if (p.is_regular_file()) {
             std::string filename = p.path().filename().string();
             if (filename.find(logPattern) != std::string::npos) {
-                // Output log content to stderr
                 std::ifstream logFile(p.path());
                 if (logFile.is_open()) {
                     std::cerr << logFile.rdbuf();
@@ -298,109 +487,107 @@ void processRelease(const std::string& RELEASE, const YAML::Node& config) {
 
     std::cout << "Building britney indexes..." << std::endl;
 
-    // Create output directory
-    fs::create_directories(fs::path(BRITNEY_OUTDIR) / BRITNEY_TIMESTAMP);
+    fs::create_directories(fs::path(BRITNEY_OUTDIR) / getCurrentTimestamp());
 
-    // "Unstable" is SOURCE_PPA
     std::string DEST = BRITNEY_DATADIR + RELEASE + "-proposed";
     fs::create_directories(DEST);
     fs::create_directories(fs::path(BRITNEY_DATADIR) / (RELEASE + "-proposed") / "state");
     writeFile(fs::path(BRITNEY_DATADIR) / (RELEASE + "-proposed") / "state" / "age-policy-dates", "");
 
-    // Create symlink for Hints
     fs::remove(fs::path(DEST) / "Hints");
     fs::create_symlink(BRITNEY_HINTDIR, fs::path(DEST) / "Hints");
 
-    // Concatenate Sources.gz files for SOURCE_PPA
-    std::string sourcesContent;
-    for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE + SOURCE_PPA + "-" + RELEASE)) {
-        if (p.path().filename() == "Sources.gz") {
-            sourcesContent += decompressGzip(p.path());
+    // Concatenate Sources from SOURCE_PPA
+    {
+        std::string sourcesContent;
+        for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE + SOURCE_PPA + "-" + RELEASE)) {
+            if (p.path().filename() == "Sources.gz") {
+                sourcesContent += decompressGzip(p.path());
+            }
         }
-    }
-    writeFile(fs::path(DEST) / "Sources", sourcesContent);
+        writeFile(fs::path(DEST) / "Sources", sourcesContent);
 
-    // Concatenate Packages.gz files for SOURCE_PPA
-    for (const auto& arch : ARCHES) {
-        std::string packagesContent;
-        for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE + SOURCE_PPA + "-" + RELEASE)) {
-            if (p.path().filename() == "Packages.gz" && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) {
-                packagesContent += decompressGzip(p.path());
+        for (const auto& arch : ARCHES) {
+            std::string packagesContent;
+            for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE + SOURCE_PPA + "-" + RELEASE)) {
+                if (p.path().filename() == "Packages.gz" && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) {
+                    packagesContent += decompressGzip(p.path());
+                }
             }
+            writeFile(fs::path(DEST) / ("Packages_" + arch), packagesContent);
         }
-        writeFile(fs::path(DEST) / ("Packages_" + arch), packagesContent);
-    }
-    for (const auto& arch : PORTS_ARCHES) {
-        std::string packagesContent;
-        for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE + SOURCE_PPA + "-" + RELEASE)) {
-            if (p.path().filename() == "Packages.gz" && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) {
-                packagesContent += decompressGzip(p.path());
+        for (const auto& arch : PORTS_ARCHES) {
+            std::string packagesContent;
+            for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE + SOURCE_PPA + "-" + RELEASE)) {
+                if (p.path().filename() == "Packages.gz" && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) {
+                    packagesContent += decompressGzip(p.path());
+                }
             }
+            writeFile(fs::path(DEST) / ("Packages_" + arch), packagesContent);
         }
-        writeFile(fs::path(DEST) / ("Packages_" + arch), packagesContent);
-    }
-
-    writeFile(fs::path(DEST) / "Blocks", "");
-    writeFile(fs::path(BRITNEY_DATADIR) / (SOURCE_PPA + "-" + RELEASE) / "Dates", "");
 
-    // Similar steps for "Testing"
-    DEST = BRITNEY_DATADIR + RELEASE;
-    fs::create_directories(DEST);
-    fs::create_directories(fs::path(BRITNEY_DATADIR) / RELEASE / "state");
-    writeFile(fs::path(BRITNEY_DATADIR) / RELEASE / "state" / "age-policy-dates", "");
-
-    fs::remove(fs::path(DEST) / "Hints");
-    fs::create_symlink(BRITNEY_HINTDIR, fs::path(DEST) / "Hints");
+        writeFile(fs::path(DEST) / "Blocks", "");
+        writeFile(fs::path(BRITNEY_DATADIR) / (SOURCE_PPA + "-" + RELEASE) / "Dates", "");
+    }
 
-    // Concatenate Sources.gz files for RELEASE
-    sourcesContent.clear();
-    for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE)) {
-        if (p.path().filename() == "Sources.gz" && p.path().string().find(RELEASE) != std::string::npos) {
-            sourcesContent += decompressGzip(p.path());
+    // Process Testing
+    {
+        DEST = BRITNEY_DATADIR + RELEASE;
+        fs::create_directories(DEST);
+        fs::create_directories(fs::path(BRITNEY_DATADIR) / RELEASE / "state");
+        writeFile(fs::path(BRITNEY_DATADIR) / RELEASE / "state" / "age-policy-dates", "");
+
+        fs::remove(fs::path(DEST) / "Hints");
+        fs::create_symlink(BRITNEY_HINTDIR, fs::path(DEST) / "Hints");
+
+        // Concatenate Sources for RELEASE
+        {
+            std::string sourcesContent;
+            for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE)) {
+                if (p.path().filename() == "Sources.gz" && p.path().string().find(RELEASE) != std::string::npos) {
+                    sourcesContent += decompressGzip(p.path());
+                }
+            }
+            writeFile(fs::path(DEST) / "Sources", sourcesContent);
+            regexReplaceInFile(fs::path(DEST) / "Sources", "Section: universe/", "Section: ");
         }
-    }
-    writeFile(fs::path(DEST) / "Sources", sourcesContent);
-    // Replace "Section: universe/" with "Section: "
-    regexReplaceInFile(fs::path(DEST) / "Sources", "Section: universe/", "Section: ");
 
-    // Concatenate Packages.gz files for RELEASE
-    for (const auto& arch : ARCHES) {
-        std::string packagesContent;
-        for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE)) {
-            if (p.path().filename() == "Packages.gz" && p.path().string().find(RELEASE) != std::string::npos && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) {
-                packagesContent += decompressGzip(p.path());
+        for (const auto& arch : ARCHES) {
+            std::string packagesContent;
+            for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE)) {
+                if (p.path().filename() == "Packages.gz" && p.path().string().find(RELEASE) != std::string::npos && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) {
+                    packagesContent += decompressGzip(p.path());
+                }
             }
+            fs::path packagesFilePath = fs::path(DEST) / ("Packages_" + arch);
+            writeFile(packagesFilePath, packagesContent);
+            regexReplaceInFile(packagesFilePath, "Section: universe/", "Section: ");
         }
-        fs::path packagesFilePath = fs::path(DEST) / ("Packages_" + arch);
-        writeFile(packagesFilePath, packagesContent);
-        // Replace "Section: universe/" with "Section: "
-        regexReplaceInFile(packagesFilePath, "Section: universe/", "Section: ");
-    }
-    for (const auto& arch : PORTS_ARCHES) {
-        std::string packagesContent;
-        for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE)) {
-            if (p.path().filename() == "Packages.gz" && p.path().string().find(RELEASE) != std::string::npos && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) {
-                packagesContent += decompressGzip(p.path());
+
+        for (const auto& arch : PORTS_ARCHES) {
+            std::string packagesContent;
+            for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE)) {
+                if (p.path().filename() == "Packages.gz" && p.path().string().find(RELEASE) != std::string::npos && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) {
+                    packagesContent += decompressGzip(p.path());
+                }
             }
+            fs::path packagesFilePath = fs::path(DEST) / ("Packages_" + arch);
+            writeFile(packagesFilePath, packagesContent);
+            regexReplaceInFile(packagesFilePath, "Section: universe/", "Section: ");
         }
-        fs::path packagesFilePath = fs::path(DEST) / ("Packages_" + arch);
-        writeFile(packagesFilePath, packagesContent);
-        // Replace "Section: universe/" with "Section: "
-        regexReplaceInFile(packagesFilePath, "Section: universe/", "Section: ");
-    }
 
-    writeFile(fs::path(DEST) / "Blocks", "");
-    writeFile(fs::path(BRITNEY_DATADIR) / (SOURCE_PPA + "-" + RELEASE) / "Dates", "");
+        writeFile(fs::path(DEST) / "Blocks", "");
+        writeFile(fs::path(BRITNEY_DATADIR) / (SOURCE_PPA + "-" + RELEASE) / "Dates", "");
+    }
 
-    // Create config file atomically
-    std::string configContent = readFile(BRITNEY_CONF);
-    // Replace variables in configContent using configuration variables
-    configContent = std::regex_replace(configContent, std::regex("%\\{SERIES\\}"), RELEASE);
-    writeFile("britney.conf", configContent);
+    // Create britney.conf
+    {
+        std::string configContent = readFile(BRITNEY_CONF);
+        configContent = std::regex_replace(configContent, std::regex("%\\{SERIES\\}"), RELEASE);
+        writeFile("britney.conf", configContent);
+    }
 
     std::cout << "Running britney..." << std::endl;
-
-    // Run britney.py
     std::string britneyCmd = BRITNEY_LOC + " -v --config britney.conf --series " + RELEASE;
     int britneyResult = executeAndLog(britneyCmd);
     if (britneyResult != 0) {
@@ -413,67 +600,66 @@ void processRelease(const std::string& RELEASE, const YAML::Node& config) {
     fs::remove_all(BRITNEY_OUTDIR);
 
     std::cout << "Moving packages..." << std::endl;
-
-    // Read candidates from HeidiResultDelta
-    std::ifstream heidiFile("output/" + RELEASE + "/HeidiResultDelta");
-    if (!heidiFile.is_open()) {
-        std::cout << "No candidates found for release " << RELEASE << "." << std::endl;
-    } else {
-        std::ofstream candidatesFile("candidates");
-        std::string line;
-        while (std::getline(heidiFile, line)) {
-            if (line.empty() || line[0] == '#') continue;
-            candidatesFile << line << std::endl;
-        }
-        heidiFile.close();
-        candidatesFile.close();
-
-        // Process candidates
-        std::ifstream candidates("candidates");
-        while (std::getline(candidates, line)) {
-            std::istringstream iss(line);
-            std::vector<std::string> packageInfo;
-            std::string word;
-            while (iss >> word) {
-                packageInfo.push_back(word);
+    {
+        std::ifstream heidiFile("output/" + RELEASE + "/HeidiResultDelta");
+        if (!heidiFile.is_open()) {
+            std::cout << "No candidates found for release " << RELEASE << "." << std::endl;
+        } else {
+            std::ofstream candidatesFile("candidates");
+            std::string line;
+            while (std::getline(heidiFile, line)) {
+                if (line.empty() || line[0] == '#') continue;
+                candidatesFile << line << std::endl;
             }
-            if (packageInfo.size() == 2) {
-                std::string COPY = "../ubuntu-archive-tools/copy-package";
-                std::string REMOVE = "../ubuntu-archive-tools/remove-package";
-                if (packageInfo[0][0] == '-') {
-                    std::string PACKAGE = packageInfo[0].substr(1);
-                    std::cout << "Demoting " << PACKAGE << "..." << std::endl;
-                    std::string copyCmd = COPY + " -y -b -s " + RELEASE + " --from ppa:" + LP_TEAM + "/ubuntu/" + DEST_PPA +
-                                          " --to ppa:" + LP_TEAM + "/ubuntu/" + SOURCE_PPA + " --version " + packageInfo[1] + " " + PACKAGE;
-                    std::string removeCmd = REMOVE + " -y -s " + RELEASE + " --archive ppa:" + LP_TEAM + "/ubuntu/" + DEST_PPA +
-                                            " --version " + packageInfo[1] + " --removal-comment=\"demoted to proposed\" " + PACKAGE;
-                    int copyResult = executeAndLog(copyCmd);
-                    if (copyResult != 0) {
-                        std::cerr << "Error: Copy command failed for package " << PACKAGE << std::endl;
-                    }
-                    int removeResult = executeAndLog(removeCmd);
-                    if (removeResult != 0) {
-                        std::cerr << "Error: Remove command failed for package " << PACKAGE << std::endl;
-                    }
-                } else {
-                    std::cout << "Migrating " << packageInfo[0] << "..." << std::endl;
-                    std::string copyCmd = COPY + " -y -b -s " + RELEASE + " --from ppa:" + LP_TEAM + "/ubuntu/" + SOURCE_PPA +
-                                          " --to ppa:" + LP_TEAM + "/ubuntu/" + DEST_PPA + " --version " + packageInfo[1] + " " + packageInfo[0];
-                    std::string removeCmd = REMOVE + " -y -s " + RELEASE + " --archive ppa:" + LP_TEAM + "/ubuntu/" + SOURCE_PPA +
-                                            " --version " + packageInfo[1] + " --removal-comment=\"moved to release\" " + packageInfo[0];
-                    int copyResult = executeAndLog(copyCmd);
-                    if (copyResult != 0) {
-                        std::cerr << "Error: Copy command failed for package " << packageInfo[0] << std::endl;
-                    }
-                    int removeResult = executeAndLog(removeCmd);
-                    if (removeResult != 0) {
-                        std::cerr << "Error: Remove command failed for package " << packageInfo[0] << std::endl;
+            heidiFile.close();
+            candidatesFile.close();
+
+            std::ifstream candidates("candidates");
+            while (std::getline(candidates, line)) {
+                std::istringstream iss(line);
+                std::vector<std::string> packageInfo;
+                std::string word;
+                while (iss >> word) {
+                    packageInfo.push_back(word);
+                }
+                if (packageInfo.size() == 2) {
+                    std::string COPY = "../ubuntu-archive-tools/copy-package";
+                    std::string REMOVE = "../ubuntu-archive-tools/remove-package";
+                    if (packageInfo[0][0] == '-') {
+                        std::string PACKAGE = packageInfo[0].substr(1);
+                        std::cout << "Demoting " << PACKAGE << "..." << std::endl;
+                        std::string copyCmd = COPY + " -y -b -s " + RELEASE + " --from ppa:" + LP_TEAM + "/ubuntu/" + DEST_PPA +
+                                              " --to ppa:" + LP_TEAM + "/ubuntu/" + SOURCE_PPA + " --version " + packageInfo[1] + " " + PACKAGE;
+                        std::string removeCmd = REMOVE + " -y -s " + RELEASE + " --archive ppa:" + LP_TEAM + "/ubuntu/" + DEST_PPA +
+                                                " --version " + packageInfo[1] + " --removal-comment=\"demoted to proposed\" " + PACKAGE;
+                        int copyResult = executeAndLog(copyCmd);
+                        if (copyResult != 0) {
+                            std::cerr << "Error: Copy command failed for package " << PACKAGE << std::endl;
+                        }
+                        int removeResult = executeAndLog(removeCmd);
+                        if (removeResult != 0) {
+                            std::cerr << "Error: Remove command failed for package " << PACKAGE << std::endl;
+                        }
+                    } else {
+                        std::cout << "Migrating " << packageInfo[0] << "..." << std::endl;
+                        std::string copyCmd = COPY + " -y -b -s " + RELEASE + " --from ppa:" + LP_TEAM + "/ubuntu/" + SOURCE_PPA +
+                                              " --to ppa:" + LP_TEAM + "/ubuntu/" + DEST_PPA + " --version " + packageInfo[1] + " " + packageInfo[0];
+                        std::string removeCmd = REMOVE + " -y -s " + RELEASE + " --archive ppa:" + LP_TEAM + "/ubuntu/" + SOURCE_PPA +
+                                                " --version " + packageInfo[1] + " --removal-comment=\"moved to release\" " + packageInfo[0];
+                        int copyResult = executeAndLog(copyCmd);
+                        if (copyResult != 0) {
+                            std::cerr << "Error: Copy command failed for package " << packageInfo[0] << std::endl;
+                        }
+                        int removeResult = executeAndLog(removeCmd);
+                        if (removeResult != 0) {
+                            std::cerr << "Error: Remove command failed for package " << packageInfo[0] << std::endl;
+                        }
                     }
                 }
             }
+            candidates.close();
+            fs::remove("candidates");
         }
-        candidates.close();
-        fs::remove("candidates");
     }
 
     std::cout << "Run the grim reaper..." << std::endl;
@@ -494,30 +680,23 @@ void printHelp(const char* programName) {
     std::cout << "  -h, --help            Show this help message and exit\n";
 }
 
-// Refresh package indexes by downloading files in parallel
 void refresh(const std::string& url, const std::string& pocket, const std::string& britneyCache, std::mutex& logMutex) {
-    // Compute directory path based on the URL
     fs::path urlPath(url);
     std::string lastTwoComponents = urlPath.parent_path().parent_path().filename().string() + "/" + urlPath.parent_path().filename().string();
     fs::path dir = fs::path(britneyCache) / pocket / lastTwoComponents;
 
-    // Create necessary directories
     fs::create_directories(dir);
 
-    // Update timestamps to prevent expiration
-    auto now = fs::file_time_type::clock::now(); // Use the same clock
+    auto now = fs::file_time_type::clock::now();
     fs::last_write_time(britneyCache, now);
     fs::last_write_time(fs::path(britneyCache) / pocket, now);
     fs::last_write_time(dir.parent_path(), now);
     fs::last_write_time(dir, now);
 
-    // Log file path (uses process ID)
     pid_t pid = getpid();
     fs::path logFilePath = dir / (std::to_string(pid) + "-wget-log");
 
-    // Output file path
     fs::path outputPath = dir / urlPath.filename();
 
-    // Download the file
     downloadFileWithTimestamping(url, outputPath, logFilePath, logMutex);
 }

pending-packages

@@ -1,103 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (C) 2024 Simon Quigley <tsimonq2@ubuntu.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-import argparse
-from datetime import datetime, timedelta, timezone
-from launchpadlib.launchpad import Launchpad
-
-print(f"Logging into Launchpad...")
-launchpad = Launchpad.login_with("pending-packages", "production", version="devel")
-
-print("Logged in. Initializing repositories...")
-ubuntu = launchpad.distributions["ubuntu"]
-lubuntu_ci = launchpad.people["lubuntu-ci"]
-regular = lubuntu_ci.getPPAByName(distribution=ubuntu, name="unstable-ci")
-proposed = lubuntu_ci.getPPAByName(distribution=ubuntu, name="unstable-ci-proposed")
-
-parser = argparse.ArgumentParser()
-parser.add_argument("release")
-args = parser.parse_args()
-series = ubuntu.getSeries(name_or_version=args.release)
-
-# First, check if any sources are still publishing
-print("Repositories initialized. Checking for pending sources...")
-records = [regular.getPublishedSources(status="Pending", distro_series=series),
-           proposed.getPublishedSources(status="Pending", distro_series=series)]
-total_pending = sum([len(i) for i in records])
-has_pending = total_pending != 0
-
-if has_pending:
-    print(f"Total sources pending: {total_pending}")
-    print("Sources are still pending, not running Britney")
-    exit(1)
-
-# Finally, check if any builds are still running/queued
-print("No pending sources, continuing. Checking for pending builds...")
-total_pending = 0
-total_retried = 0
-for archive in [proposed, regular]:
-    one_hour_ago = datetime.now(timezone.utc) - timedelta(hours=1)
-    for source in archive.getPublishedSources(status="Published", distro_series=series):
-        for build in source.getBuilds():
-            if build.buildstate == "Currently building":
-                if build.date_started >= one_hour_ago:
-                    total_pending += 1
-            elif build.buildstate == "Needs building":
-                total_pending += 1
-            # This isn't technically related, but retry failed builds without logs
-            elif build.buildstate == "Chroot problem" or (build.buildstate == "Failed to build" and not build.build_log_url):
-                if build.can_be_retried:
-                    build.retry()
-                    total_pending += 1
-                    total_retried += 1
-
-if total_retried != 0:
-    print(f"Total builds retried due to builder flakiness: {total_retried}")
-
-if total_pending != 0:
-    print(f"Total builds pending: {total_pending}")
-    print("Builds are still running, not running Britney")
-    exit(1)
-
-print("No pending builds, continuing. Checking for pending binaries...")
-has_pending = False
-for pocket in [proposed, regular]:
-    if has_pending:
-        break
-    three_hours_ago = datetime.now(timezone.utc) - timedelta(hours=3)
-    check_builds = set()
-    current_builds = set()
-    source_packages = []
-
-    for build in pocket.getBuildRecords(build_state="Successfully built"):
-        if build.datebuilt < three_hours_ago:
-            del source_packages
-            break
-        check_builds.add(build.title)
-        source_package = build.current_source_publication
-        if source_package and source_package.distro_series == series and source_package not in source_packages:
-            source_packages.append(source_package)
-            for binary in source_package.getPublishedBinaries():
-                current_builds.add(binary.build.title)
-
-    has_pending = not check_builds.issuperset(current_builds) or has_pending
-
-if has_pending:
-    print("Binaries are still pending, not running Britney")
-    exit(1)
-
-print("All clear. Starting Britney.")