From 706eb2558112b85dbc65b3afa2de2eabfca67d00 Mon Sep 17 00:00:00 2001 From: Simon Quigley Date: Wed, 4 Dec 2024 02:28:33 -0600 Subject: [PATCH] Convert fetch-indexes to C++ --- .gitignore | 2 + configs/fetch-indexes.yaml | 28 ++ fetch-indexes | 161 ----------- fetch-indexes-cpp/CMakeLists.txt | 17 ++ fetch-indexes-cpp/main.cpp | 482 +++++++++++++++++++++++++++++++ fetch-indexes-cpp/utilities.cpp | 132 +++++++++ fetch-indexes-cpp/utilities.h | 42 +++ pull-and-compile-tooling | 8 + run-britney | 37 --- systemd/britney.service | 2 +- systemd/pull-tooling.service | 11 + systemd/pull-tooling.timer | 10 + 12 files changed, 733 insertions(+), 199 deletions(-) create mode 100644 configs/fetch-indexes.yaml delete mode 100755 fetch-indexes create mode 100644 fetch-indexes-cpp/CMakeLists.txt create mode 100644 fetch-indexes-cpp/main.cpp create mode 100644 fetch-indexes-cpp/utilities.cpp create mode 100644 fetch-indexes-cpp/utilities.h create mode 100755 pull-and-compile-tooling delete mode 100755 run-britney create mode 100644 systemd/pull-tooling.service create mode 100644 systemd/pull-tooling.timer diff --git a/.gitignore b/.gitignore index bee8a64..c4d1968 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ __pycache__ +fetch-indexes +fetch-indexes-cpp/build diff --git a/configs/fetch-indexes.yaml b/configs/fetch-indexes.yaml new file mode 100644 index 0000000..e744534 --- /dev/null +++ b/configs/fetch-indexes.yaml @@ -0,0 +1,28 @@ +MAIN_ARCHIVE: "http://us.archive.ubuntu.com/ubuntu/dists/" +PORTS_ARCHIVE: "http://us.ports.ubuntu.com/ubuntu-ports/dists/" +LP_TEAM: "lubuntu-ci" +SOURCE_PPA: "unstable-ci-proposed" +DEST_PPA: "unstable-ci" + +ARCHES: + - "amd64" +PORTS_ARCHES: + - "arm64" + - "armhf" + - "ppc64el" + - "riscv64" + - "s390x" + +BRITNEY_CACHE: "cache/" +BRITNEY_DATADIR: "data/" +BRITNEY_OUTDIR: "output/" +BRITNEY_HINTDIR: "../hints-ubuntu/" +BRITNEY_LOC: "/srv/lubuntu-ci/repos/britney2-ubuntu/britney.py" + +LOG_DIR: "/srv/lubuntu-ci/output/logs/britney" +MAX_LOG_AGE_DAYS: 1 + +RELEASES: + - "plucky" + - "oracular" + - "noble" diff --git a/fetch-indexes b/fetch-indexes deleted file mode 100755 index 39ad300..0000000 --- a/fetch-indexes +++ /dev/null @@ -1,161 +0,0 @@ -#!/bin/bash -# download current package indexes to data/{,-proposed}/ for running -# britney against a PPA. The PPA will play the role of "-proposed" (i. e. -# "unstable" in britney terms, containing the updated packages to test), the -# Ubuntu archive has the "-release" part (i. e. "testing" in britney terms, in -# which the -proposed packages are being landed). -# -# Copyright (C) 2019-2024 Simon Quigley -# Copyright (C) Canonical Ltd -# Author: Martin Pitt -# Author: Robert Bruce Park -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -set -u - -./pending-packages $RELEASE || exit 0 - -export MAIN_ARCHIVE="http://us.archive.ubuntu.com/ubuntu/dists/" -export PORTS_ARCHIVE="http://us.ports.ubuntu.com/ubuntu-ports/dists/" -export LP_TEAM="lubuntu-ci" -export SOURCE_PPA="unstable-ci-proposed" -export DEST_PPA="unstable-ci" -export SOURCE_PPA_URL="https://ppa.launchpadcontent.net/$LP_TEAM/$SOURCE_PPA/ubuntu/dists/$RELEASE/main"; -export DEST_PPA_URL="https://ppa.launchpadcontent.net/$LP_TEAM/$DEST_PPA/ubuntu/dists/$RELEASE/main"; -export ARCHES="amd64" -export PORTS_ARCHES="arm64 armhf ppc64el riscv64 s390x" -export BRITNEY_CACHE="cache/" -export BRITNEY_DATADIR="data/" -export BRITNEY_OUTDIR="output/" -export BRITNEY_HINTDIR="../hints-ubuntu/" -export BRITNEY_LOC="/srv/lubuntu-ci/repos/britney2-ubuntu/britney.py" -export BRITNEY_TIMESTAMP=$(date +"%Y-%m-%d_%H:%M:%S") - -echo "Release: $RELEASE"; -echo "Timestamp: $BRITNEY_TIMESTAMP" - -# Download files in parallel in background, only if there is an update -refresh() { - DIR=$BRITNEY_CACHE/$pocket/$(echo $1 | rev | cut --delimiter=/ --fields=2,3 | rev) - mkdir --parents $DIR - touch --no-create $BRITNEY_CACHE $BRITNEY_CACHE/$pocket "$(dirname $DIR)" $DIR # Timestamp thwarts expire.sh - wget --directory-prefix $DIR --timestamping $1 --append-output $DIR/$$-wget-log --no-verbose & -} - -echo 'Refreshing package indexes...' - -for pocket in $RELEASE $RELEASE-updates; do - for component in main restricted universe multiverse; do - for arch in $ARCHES; do - refresh $MAIN_ARCHIVE/$pocket/$component/binary-$arch/Packages.gz - done - for arch in $PORTS_ARCHES; do - refresh $PORTS_ARCHIVE/$pocket/$component/binary-$arch/Packages.gz - done - refresh $MAIN_ARCHIVE/$pocket/$component/source/Sources.gz - done -done - -# Treat the destination PPA as just another pocket -for pocket in $RELEASE-ppa-proposed; do - for arch in $ARCHES $PORTS_ARCHES; do - refresh $DEST_PPA_URL/source/Sources.gz - refresh $DEST_PPA_URL/binary-$arch/Packages.gz - done -done - -# Get the source PPA -pocket=$SOURCE_PPA-$RELEASE -for arch in $ARCHES $PORTS_ARCHES; do - refresh $SOURCE_PPA_URL/binary-$arch/Packages.gz -done -refresh $SOURCE_PPA_URL/source/Sources.gz - -wait # for wgets to finish - -find $BRITNEY_DATADIR -name "$$-wget-log*" -exec cat '{}' \; -delete 1>&2 - -echo 'Building britney indexes...' - -mkdir --parents "$BRITNEY_OUTDIR/$BRITNEY_TIMESTAMP/" - -# "Unstable" is SOURCE_PPA -DEST=$BRITNEY_DATADIR/$RELEASE-proposed -mkdir --parents $DEST -mkdir -pv $BRITNEY_DATADIR/$RELEASE-proposed/state/ -touch $BRITNEY_DATADIR/$RELEASE-proposed/state/age-policy-dates -touch --no-create $DEST -ln --verbose --symbolic --force --no-dereference $BRITNEY_HINTDIR $DEST/Hints -zcat $BRITNEY_CACHE/$SOURCE_PPA-$RELEASE/*/source/Sources.gz > $DEST/Sources -for arch in $ARCHES $PORTS_ARCHES; do - zcat $BRITNEY_CACHE/$SOURCE_PPA-$RELEASE/*/binary-$arch/Packages.gz > $DEST/Packages_${arch} -done -touch $DEST/Blocks $DEST/Dates - -# "Testing" is a combination of the archive and DEST_PPA -DEST=$BRITNEY_DATADIR/$RELEASE -mkdir --parents $DEST -mkdir -pv $BRITNEY_DATADIR/$RELEASE/state/ -touch $BRITNEY_DATADIR/$RELEASE/state/age-policy-dates -touch --no-create $DEST -ln --verbose --symbolic --force --no-dereference $BRITNEY_HINTDIR $DEST/Hints -zcat $BRITNEY_CACHE/$RELEASE*/*/source/Sources.gz > $DEST/Sources -sed -i "s/Section: universe\//Section: /g" $DEST/Sources -for arch in $ARCHES $PORTS_ARCHES; do - zcat $BRITNEY_CACHE/$RELEASE*/*/binary-$arch/Packages.gz > $DEST/Packages_${arch} - sed -i "s/Section: universe\//Section: /g" $DEST/Packages_${arch} -done -touch $DEST/Blocks -touch "$BRITNEY_DATADIR/$SOURCE_PPA-$RELEASE/Dates" - -# Create config file atomically. -CONFIG="britney.conf" -cp $CONFIG $CONFIG.bak -envsubst < "$CONFIG.bak" > "$CONFIG" -rm $CONFIG.bak - -echo 'Running britney...' -$BRITNEY_LOC -v --config "$CONFIG" --series $RELEASE - -echo 'Syncing output to frontend...' -rmdir output/; -rsync -da output/ ../../output/britney - -echo "$0 done." - -echo "Moving packages..." - -egrep -v '^#' output/$RELEASE/HeidiResultDelta > candidates || echo "No candidates found."; - -while read -r -a package; do - # This only acts on sources; binaries require manual cleanup - if [ ${#package[@]} = 2 ]; then - COPY="../ubuntu-archive-tools/copy-package" - REMOVE="../ubuntu-archive-tools/remove-package" - if echo ${package[0]} | egrep -q "^-"; then - PACKAGE=$(echo ${package[0]} | sed 's/-//') - echo "Demoting $PACKAGE..." - $COPY -y -b -s $RELEASE --from "ppa:$LP_TEAM/ubuntu/$DEST_PPA" --to "ppa:$LP_TEAM/ubuntu/$SOURCE_PPA" --version "${package[1]}" "$PACKAGE"; - $REMOVE -y -s $RELEASE --archive "ppa:$LP_TEAM/ubuntu/$DEST_PPA" --version "${package[1]}" --removal-comment="demoted to proposed" "$PACKAGE"; - else - echo "Migrating ${package[0]}..." - $COPY -y -b -s $RELEASE --from "ppa:$LP_TEAM/ubuntu/$SOURCE_PPA" --to "ppa:$LP_TEAM/ubuntu/$DEST_PPA" --version "${package[1]}" "${package[0]}"; - $REMOVE -y -s $RELEASE --archive "ppa:$LP_TEAM/ubuntu/$SOURCE_PPA" --version "${package[1]}" --removal-comment="moved to release" "${package[0]}"; - fi - fi -done < candidates; -rm candidates; - -echo "Run the grim reaper..." -./grim-reaper $RELEASE diff --git a/fetch-indexes-cpp/CMakeLists.txt b/fetch-indexes-cpp/CMakeLists.txt new file mode 100644 index 0000000..b4b826c --- /dev/null +++ b/fetch-indexes-cpp/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 3.10) +project(fetch-indexes) + +set(CMAKE_CXX_STANDARD 23) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +find_package(CURL REQUIRED) +find_package(ZLIB REQUIRED) +find_package(YAML-CPP REQUIRED) + +include_directories(${CURL_INCLUDE_DIRS}) +include_directories(${ZLIB_INCLUDE_DIRS}) +include_directories(${YAML_CPP_INCLUDE_DIR}) + +add_executable(fetch-indexes main.cpp utilities.cpp) + +target_link_libraries(fetch-indexes ${CURL_LIBRARIES} ${ZLIB_LIBRARIES} yaml-cpp) diff --git a/fetch-indexes-cpp/main.cpp b/fetch-indexes-cpp/main.cpp new file mode 100644 index 0000000..111c433 --- /dev/null +++ b/fetch-indexes-cpp/main.cpp @@ -0,0 +1,482 @@ +// Copyright (C) 2024 Simon Quigley +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utilities.h" + +namespace fs = std::filesystem; + +// Function prototypes +void printHelp(const char* programName); +void processRelease(const std::string& release, const YAML::Node& config); +void refresh(const std::string& url, const std::string& pocket, const std::string& britneyCache, std::mutex& logMutex); + +int main(int argc, char* argv[]) { + std::string configFilePath = "config.yaml"; + + // Command-line argument parsing + int opt; + bool showHelp = false; + + static struct option long_options[] = { + {"config", required_argument, 0, 'c'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0 } + }; + + while (true) { + int option_index = 0; + opt = getopt_long(argc, argv, "c:h", long_options, &option_index); + + if (opt == -1) + break; + + switch (opt) { + case 'c': + configFilePath = optarg; + break; + case 'h': + showHelp = true; + break; + default: + // Unknown option + showHelp = true; + break; + } + } + + if (showHelp) { + printHelp(argv[0]); + return 0; + } + + // Load configuration from YAML file + YAML::Node config; + try { + config = YAML::LoadFile(configFilePath); + } catch (const YAML::BadFile& e) { + std::cerr << "Error: Unable to open config file: " << configFilePath << std::endl; + return 1; + } catch (const YAML::ParserException& e) { + std::cerr << "Error: Failed to parse config file: " << e.what() << std::endl; + return 1; + } + + // Ensure LOG_DIR exists + std::string LOG_DIR = config["LOG_DIR"].as(); + fs::create_directories(LOG_DIR); + + // Log rotation: Remove logs older than MAX_LOG_AGE_DAYS + int maxLogAgeDays = config["MAX_LOG_AGE_DAYS"].as(); + auto now = fs::file_time_type::clock::now(); // Use the same clock as file_time_type + + for (const auto& entry : fs::directory_iterator(LOG_DIR)) { + if (entry.is_regular_file()) { + auto ftime = fs::last_write_time(entry.path()); + auto age = std::chrono::duration_cast(now - ftime).count() / 24; + if (age > maxLogAgeDays) { + fs::remove(entry.path()); + } + } + } + + // Get the list of releases + std::vector releases = config["RELEASES"].as>(); + + // Process each release + for (const auto& release : releases) { + // Log file named by current UTC time (YYYYMMDDTHH:MM:SS) and release + std::time_t now_c = std::time(nullptr); + char timestamp[20]; + std::strftime(timestamp, sizeof(timestamp), "%Y%m%dT%H:%M:%S", std::gmtime(&now_c)); + std::string logFileName = LOG_DIR + "/" + timestamp + "_" + release + ".log"; + + // Open log file + std::ofstream logFile(logFileName, std::ios::app); + if (!logFile.is_open()) { + std::cerr << "Error: Unable to open log file: " << logFileName << std::endl; + continue; + } + + // Redirect stdout and stderr to log file + std::streambuf* coutBuf = std::cout.rdbuf(); + std::streambuf* cerrBuf = std::cerr.rdbuf(); + std::cout.rdbuf(logFile.rdbuf()); + std::cerr.rdbuf(logFile.rdbuf()); + + // Log the start time + char startTime[20]; + std::strftime(startTime, sizeof(startTime), "%Y-%m-%d %H:%M:%S", std::gmtime(&now_c)); + std::cout << startTime << " - Running Britney for " << release << std::endl; + + // Process the release + processRelease(release, config); + + // Restore stdout and stderr + std::cout.rdbuf(coutBuf); + std::cerr.rdbuf(cerrBuf); + + // Close log file + logFile.close(); + } + + return 0; +} + +void processRelease(const std::string& RELEASE, const YAML::Node& config) { + // Extract configuration variables + std::string MAIN_ARCHIVE = config["MAIN_ARCHIVE"].as(); + std::string PORTS_ARCHIVE = config["PORTS_ARCHIVE"].as(); + std::string LP_TEAM = config["LP_TEAM"].as(); + std::string SOURCE_PPA = config["SOURCE_PPA"].as(); + std::string DEST_PPA = config["DEST_PPA"].as(); + std::string BRITNEY_CACHE = config["BRITNEY_CACHE"].as(); + std::string BRITNEY_DATADIR = config["BRITNEY_DATADIR"].as(); + std::string BRITNEY_OUTDIR = config["BRITNEY_OUTDIR"].as(); + std::string BRITNEY_HINTDIR = config["BRITNEY_HINTDIR"].as(); + std::string BRITNEY_LOC = config["BRITNEY_LOC"].as(); + + std::vector ARCHES = config["ARCHES"].as>(); + std::vector PORTS_ARCHES = config["PORTS_ARCHES"].as>(); + + std::string SOURCE_PPA_URL = "https://ppa.launchpadcontent.net/" + LP_TEAM + "/" + SOURCE_PPA + "/ubuntu/dists/" + RELEASE + "/main"; + std::string DEST_PPA_URL = "https://ppa.launchpadcontent.net/" + LP_TEAM + "/" + DEST_PPA + "/ubuntu/dists/" + RELEASE + "/main"; + + // Get current timestamp + std::time_t now_c = std::time(nullptr); + char timestamp[20]; + std::strftime(timestamp, sizeof(timestamp), "%Y-%m-%d_%H:%M:%S", std::localtime(&now_c)); + std::string BRITNEY_TIMESTAMP(timestamp); + + std::cout << "Release: " << RELEASE << std::endl; + std::cout << "Timestamp: " << BRITNEY_TIMESTAMP << std::endl; + + // Execute pending-packages script + std::string pendingCmd = "./pending-packages " + RELEASE; + int pendingResult = std::system(pendingCmd.c_str()); + if (pendingResult != 0) { + std::cerr << "Error: pending-packages script failed for release " << RELEASE << std::endl; + return; + } + + std::cout << "Refreshing package indexes..." << std::endl; + + std::vector threads; + std::mutex logMutex; + + // Refresh package indexes + std::vector pockets = {RELEASE, RELEASE + "-updates"}; + std::vector components = {"main", "restricted", "universe", "multiverse"}; + + // Loop over pockets, components, architectures to refresh package indexes + for (const auto& pocket : pockets) { + for (const auto& component : components) { + for (const auto& arch : ARCHES) { + std::string url = MAIN_ARCHIVE + pocket + "/" + component + "/binary-" + arch + "/Packages.gz"; + threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex)); + } + for (const auto& arch : PORTS_ARCHES) { + std::string url = PORTS_ARCHIVE + pocket + "/" + component + "/binary-" + arch + "/Packages.gz"; + threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex)); + } + std::string url = MAIN_ARCHIVE + pocket + "/" + component + "/source/Sources.gz"; + threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex)); + } + } + + // Treat the destination PPA as just another pocket + std::string pocket = RELEASE + "-ppa-proposed"; + for (const auto& arch : ARCHES) { + std::string url = DEST_PPA_URL + "/binary-" + arch + "/Packages.gz"; + threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex)); + } + for (const auto& arch : PORTS_ARCHES) { + std::string url = DEST_PPA_URL + "/binary-" + arch + "/Packages.gz"; + threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex)); + } + { + std::string url = DEST_PPA_URL + "/source/Sources.gz"; + threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex)); + } + + // Get the source PPA + pocket = SOURCE_PPA + "-" + RELEASE; + for (const auto& arch : ARCHES) { + std::string url = SOURCE_PPA_URL + "/binary-" + arch + "/Packages.gz"; + threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex)); + } + for (const auto& arch : PORTS_ARCHES) { + std::string url = SOURCE_PPA_URL + "/binary-" + arch + "/Packages.gz"; + threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex)); + } + { + std::string url = SOURCE_PPA_URL + "/source/Sources.gz"; + threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex)); + } + + // Wait for all threads to finish + for (auto& th : threads) { + th.join(); + } + + // Process logs and delete them + pid_t pid = getpid(); + std::string logPattern = std::to_string(pid) + "-wget-log"; + + for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE)) { + if (p.is_regular_file()) { + std::string filename = p.path().filename().string(); + if (filename.find(logPattern) != std::string::npos) { + // Output log content to stderr + std::ifstream logFile(p.path()); + std::cerr << logFile.rdbuf(); + logFile.close(); + fs::remove(p.path()); + } + } + } + + std::cout << "Building britney indexes..." << std::endl; + + // Create output directory + fs::create_directories(fs::path(BRITNEY_OUTDIR) / BRITNEY_TIMESTAMP); + + // "Unstable" is SOURCE_PPA + std::string DEST = BRITNEY_DATADIR + RELEASE + "-proposed"; + fs::create_directories(DEST); + fs::create_directories(fs::path(BRITNEY_DATADIR) / (RELEASE + "-proposed") / "state"); + writeFile(fs::path(BRITNEY_DATADIR) / (RELEASE + "-proposed") / "state" / "age-policy-dates", ""); + + // Create symlink for Hints + fs::remove(fs::path(DEST) / "Hints"); + fs::create_symlink(BRITNEY_HINTDIR, fs::path(DEST) / "Hints"); + + // Concatenate Sources.gz files for SOURCE_PPA + std::string sourcesContent; + for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE + SOURCE_PPA + "-" + RELEASE)) { + if (p.path().filename() == "Sources.gz") { + sourcesContent += decompressGzip(p.path()); + } + } + writeFile(fs::path(DEST) / "Sources", sourcesContent); + + // Concatenate Packages.gz files for SOURCE_PPA + for (const auto& arch : ARCHES) { + std::string packagesContent; + for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE + SOURCE_PPA + "-" + RELEASE)) { + if (p.path().filename() == "Packages.gz" && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) { + packagesContent += decompressGzip(p.path()); + } + } + writeFile(fs::path(DEST) / ("Packages_" + arch), packagesContent); + } + for (const auto& arch : PORTS_ARCHES) { + std::string packagesContent; + for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE + SOURCE_PPA + "-" + RELEASE)) { + if (p.path().filename() == "Packages.gz" && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) { + packagesContent += decompressGzip(p.path()); + } + } + writeFile(fs::path(DEST) / ("Packages_" + arch), packagesContent); + } + + writeFile(fs::path(DEST) / "Blocks", ""); + writeFile(fs::path(DEST) / "Dates", ""); + + // Similar steps for "Testing" + DEST = BRITNEY_DATADIR + RELEASE; + fs::create_directories(DEST); + fs::create_directories(fs::path(BRITNEY_DATADIR) / RELEASE / "state"); + writeFile(fs::path(BRITNEY_DATADIR) / RELEASE / "state" / "age-policy-dates", ""); + + fs::remove(fs::path(DEST) / "Hints"); + fs::create_symlink(BRITNEY_HINTDIR, fs::path(DEST) / "Hints"); + + // Concatenate Sources.gz files for RELEASE + sourcesContent.clear(); + for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE)) { + if (p.path().filename() == "Sources.gz" && p.path().string().find(RELEASE) != std::string::npos) { + sourcesContent += decompressGzip(p.path()); + } + } + writeFile(fs::path(DEST) / "Sources", sourcesContent); + // Replace "Section: universe/" with "Section: " + regexReplaceInFile(fs::path(DEST) / "Sources", "Section: universe/", "Section: "); + + // Concatenate Packages.gz files for RELEASE + for (const auto& arch : ARCHES) { + std::string packagesContent; + for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE)) { + if (p.path().filename() == "Packages.gz" && p.path().string().find(RELEASE) != std::string::npos && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) { + packagesContent += decompressGzip(p.path()); + } + } + fs::path packagesFilePath = fs::path(DEST) / ("Packages_" + arch); + writeFile(packagesFilePath, packagesContent); + // Replace "Section: universe/" with "Section: " + regexReplaceInFile(packagesFilePath, "Section: universe/", "Section: "); + } + for (const auto& arch : PORTS_ARCHES) { + std::string packagesContent; + for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE)) { + if (p.path().filename() == "Packages.gz" && p.path().string().find(RELEASE) != std::string::npos && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) { + packagesContent += decompressGzip(p.path()); + } + } + fs::path packagesFilePath = fs::path(DEST) / ("Packages_" + arch); + writeFile(packagesFilePath, packagesContent); + // Replace "Section: universe/" with "Section: " + regexReplaceInFile(packagesFilePath, "Section: universe/", "Section: "); + } + + writeFile(fs::path(DEST) / "Blocks", ""); + writeFile(fs::path(BRITNEY_DATADIR) / (SOURCE_PPA + "-" + RELEASE) / "Dates", ""); + + // Create config file atomically + std::string configContent = readFile("britney.conf"); + // Replace variables in configContent using configuration variables + configContent = std::regex_replace(configContent, std::regex("\\$\\{RELEASE\\}"), RELEASE); + configContent = std::regex_replace(configContent, std::regex("\\$\\{BRITNEY_DATADIR\\}"), BRITNEY_DATADIR); + writeFile("britney.conf", configContent); + + std::cout << "Running britney..." << std::endl; + + // Run britney.py + std::string britneyCmd = BRITNEY_LOC + " -v --config britney.conf --series " + RELEASE; + int britneyResult = std::system(britneyCmd.c_str()); + if (britneyResult != 0) { + std::cerr << "Error: Britney execution failed for release " << RELEASE << std::endl; + return; + } + + std::cout << "Syncing output to frontend..." << std::endl; + // Rsync command can be replaced with filesystem copy + fs::remove_all("output"); + fs::copy("output", "../../output/britney", fs::copy_options::recursive | fs::copy_options::overwrite_existing); + + std::cout << "Moving packages..." << std::endl; + + // Read candidates from HeidiResultDelta + std::ifstream heidiFile("output/" + RELEASE + "/HeidiResultDelta"); + if (!heidiFile.is_open()) { + std::cout << "No candidates found for release " << RELEASE << "." << std::endl; + } else { + std::ofstream candidatesFile("candidates"); + std::string line; + while (std::getline(heidiFile, line)) { + if (line.empty() || line[0] == '#') continue; + candidatesFile << line << std::endl; + } + heidiFile.close(); + candidatesFile.close(); + + // Process candidates + std::ifstream candidates("candidates"); + while (std::getline(candidates, line)) { + std::istringstream iss(line); + std::vector packageInfo; + std::string word; + while (iss >> word) { + packageInfo.push_back(word); + } + if (packageInfo.size() == 2) { + std::string COPY = "../ubuntu-archive-tools/copy-package"; + std::string REMOVE = "../ubuntu-archive-tools/remove-package"; + if (packageInfo[0][0] == '-') { + std::string PACKAGE = packageInfo[0].substr(1); + std::cout << "Demoting " << PACKAGE << "..." << std::endl; + std::string copyCmd = COPY + " -y -b -s " + RELEASE + " --from ppa:" + LP_TEAM + "/ubuntu/" + DEST_PPA + + " --to ppa:" + LP_TEAM + "/ubuntu/" + SOURCE_PPA + " --version " + packageInfo[1] + " " + PACKAGE; + std::string removeCmd = REMOVE + " -y -s " + RELEASE + " --archive ppa:" + LP_TEAM + "/ubuntu/" + DEST_PPA + + " --version " + packageInfo[1] + " --removal-comment=\"demoted to proposed\" " + PACKAGE; + std::system(copyCmd.c_str()); + std::system(removeCmd.c_str()); + } else { + std::cout << "Migrating " << packageInfo[0] << "..." << std::endl; + std::string copyCmd = COPY + " -y -b -s " + RELEASE + " --from ppa:" + LP_TEAM + "/ubuntu/" + SOURCE_PPA + + " --to ppa:" + LP_TEAM + "/ubuntu/" + DEST_PPA + " --version " + packageInfo[1] + " " + packageInfo[0]; + std::string removeCmd = REMOVE + " -y -s " + RELEASE + " --archive ppa:" + LP_TEAM + "/ubuntu/" + SOURCE_PPA + + " --version " + packageInfo[1] + " --removal-comment=\"moved to release\" " + packageInfo[0]; + std::system(copyCmd.c_str()); + std::system(removeCmd.c_str()); + } + } + } + candidates.close(); + fs::remove("candidates"); + } + + std::cout << "Run the grim reaper..." << std::endl; + std::string grimCmd = "./grim-reaper " + RELEASE; + int grimResult = std::system(grimCmd.c_str()); + if (grimResult != 0) { + std::cerr << "Error: Grim reaper execution failed for release " << RELEASE << std::endl; + return; + } + + std::cout << "Done processing release " << RELEASE << "." << std::endl; +} + +void printHelp(const char* programName) { + std::cout << "Usage: " << programName << " [options]\n"; + std::cout << "Options:\n"; + std::cout << " -c, --config Specify path to config file (default: config.yaml)\n"; + std::cout << " -h, --help Show this help message and exit\n"; +} + +// Refresh package indexes by downloading files in parallel +void refresh(const std::string& url, const std::string& pocket, const std::string& britneyCache, std::mutex& logMutex) { + // Compute directory path based on the URL + fs::path urlPath(url); + std::string lastTwoComponents = urlPath.parent_path().parent_path().filename().string() + "/" + urlPath.parent_path().filename().string(); + fs::path dir = fs::path(britneyCache) / pocket / lastTwoComponents; + + // Create necessary directories + fs::create_directories(dir); + + // Update timestamps to prevent expiration + auto now = fs::file_time_type::clock::now(); // Use the same clock + fs::last_write_time(britneyCache, now); + fs::last_write_time(fs::path(britneyCache) / pocket, now); + fs::last_write_time(dir.parent_path(), now); + fs::last_write_time(dir, now); + + // Log file path (uses process ID) + pid_t pid = getpid(); + fs::path logFilePath = dir / (std::to_string(pid) + "-wget-log"); + + // Output file path + fs::path outputPath = dir / urlPath.filename(); + + // Download the file + downloadFileWithTimestamping(url, outputPath, logFilePath, logMutex); +} diff --git a/fetch-indexes-cpp/utilities.cpp b/fetch-indexes-cpp/utilities.cpp new file mode 100644 index 0000000..e605cd8 --- /dev/null +++ b/fetch-indexes-cpp/utilities.cpp @@ -0,0 +1,132 @@ +// Copyright (C) 2024 Simon Quigley +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#include "utilities.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +// Function to read the entire content of a file into a string +std::string readFile(const fs::path& filePath) { + std::ifstream inFile(filePath, std::ios::binary); + if (inFile) { + return std::string((std::istreambuf_iterator(inFile)), + std::istreambuf_iterator()); + } + return ""; +} + +// Function to write a string into a file +void writeFile(const fs::path& filePath, const std::string& content) { + std::ofstream outFile(filePath, std::ios::binary); + if (outFile) { + outFile << content; + } +} + +// Function to perform in-place regex replace on a file +void regexReplaceInFile(const fs::path& filePath, const std::string& pattern, const std::string& replace) { + std::string content = readFile(filePath); + content = std::regex_replace(content, std::regex(pattern), replace); + writeFile(filePath, content); +} + +// Function to decompress gzipped files +std::string decompressGzip(const fs::path& filePath) { + gzFile infile = gzopen(filePath.c_str(), "rb"); + if (!infile) return ""; + + std::string decompressedData; + char buffer[8192]; + int numRead = 0; + while ((numRead = gzread(infile, buffer, sizeof(buffer))) > 0) { + decompressedData.append(buffer, numRead); + } + gzclose(infile); + return decompressedData; +} + +// Helper function for libcurl write callback +size_t write_data(void* ptr, size_t size, size_t nmemb, void* stream) { + FILE* out = static_cast(stream); + return fwrite(ptr, size, nmemb, out); +} + +// Function to download a file with timestamping using libcurl +void downloadFileWithTimestamping(const std::string& url, const fs::path& outputPath, + const fs::path& logFilePath, std::mutex& logMutex) { + CURL* curl; + CURLcode res; + FILE* fp; + curl = curl_easy_init(); + if (curl) { + fs::path tempFilePath = outputPath.string() + ".tmp"; + fp = fopen(tempFilePath.c_str(), "wb"); + + if (!fp) { + std::cerr << "Failed to open file: " << tempFilePath << std::endl; + curl_easy_cleanup(curl); + return; + } + + // Set curl options for downloading the file + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); + + // Timestamping: set If-Modified-Since header + struct stat file_info; + if (stat(outputPath.c_str(), &file_info) == 0) { + // Set the time condition to If-Modified-Since + curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); + curl_easy_setopt(curl, CURLOPT_TIMEVALUE, file_info.st_mtime); + } + + // Perform the file download + res = curl_easy_perform(curl); + + // Get the HTTP response code + long response_code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + + fclose(fp); + curl_easy_cleanup(curl); + + // Log the result and handle the downloaded file + { + std::lock_guard lock(logMutex); + std::ofstream logFile(logFilePath, std::ios::app); + if (res == CURLE_OK && (response_code == 200 || response_code == 201)) { + fs::rename(tempFilePath, outputPath); + logFile << "Downloaded: " << url << std::endl; + } else if (response_code == 304) { + fs::remove(tempFilePath); + logFile << "Not Modified: " << url << std::endl; + } else { + fs::remove(tempFilePath); + logFile << "Failed to download: " << url << std::endl; + } + } + } else { + std::cerr << "Failed to initialize CURL." << std::endl; + } +} diff --git a/fetch-indexes-cpp/utilities.h b/fetch-indexes-cpp/utilities.h new file mode 100644 index 0000000..58d0fc8 --- /dev/null +++ b/fetch-indexes-cpp/utilities.h @@ -0,0 +1,42 @@ +// Copyright (C) 2024 Simon Quigley +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#ifndef UTILITIES_H +#define UTILITIES_H + +#include +#include +#include + +// Function to read the entire content of a file into a string +std::string readFile(const std::filesystem::path& filePath); + +// Function to write a string into a file +void writeFile(const std::filesystem::path& filePath, const std::string& content); + +// Function to perform in-place regex replace on a file +void regexReplaceInFile(const std::filesystem::path& filePath, const std::string& pattern, const std::string& replace); + +// Function to decompress gzipped files +std::string decompressGzip(const std::filesystem::path& filePath); + +// Function to download a file with timestamping using libcurl +void downloadFileWithTimestamping(const std::string& url, const std::filesystem::path& outputPath, + const std::filesystem::path& logFilePath, std::mutex& logMutex); + +// Helper function for libcurl write callback +size_t write_data(void* ptr, size_t size, size_t nmemb, void* stream); + +#endif // UTILITIES_H diff --git a/pull-and-compile-tooling b/pull-and-compile-tooling new file mode 100755 index 0000000..c3f041f --- /dev/null +++ b/pull-and-compile-tooling @@ -0,0 +1,8 @@ +#!/bin/bash + +git pull + +(cd fetch-indexes-cpp && +rm -rf build; mkdir build && (cd build && +cmake .. && make -j$(nproc)) && +mv build/fetch-indexes ../ && rm -rf build) diff --git a/run-britney b/run-britney deleted file mode 100755 index 3fcb71f..0000000 --- a/run-britney +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2024 Simon Quigley -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Configuration -LOG_DIR="/srv/lubuntu-ci/output/logs/britney" -SCRIPT_PATH="/srv/lubuntu-ci/repos/ci-tools/fetch-indexes" - -# Ensure the log directory exists -mkdir -p $LOG_DIR - -# Log rotation: Remove logs older than MAX_LOG_AGE -find $LOG_DIR -type f -mtime +1 -exec rm -f {} \; - -# Execute the fetch-indexes script for each release and log output -for release in plucky oracular noble; do - export RELEASE="$release" - - # Log file named by current UTC time (HH-MM-SS) - LOG_FILE="$LOG_DIR/$(date -u +"%Y%m%dT%H:%M:%S")_${release}.log" - - echo "$(date -u +"%Y-%m-%d %H:%M:%S") - Running Britney for $RELEASE" >> "$LOG_FILE" - "$SCRIPT_PATH" >> "$LOG_FILE" 2>&1 -done diff --git a/systemd/britney.service b/systemd/britney.service index 7c601f3..2896517 100644 --- a/systemd/britney.service +++ b/systemd/britney.service @@ -8,4 +8,4 @@ Type=simple User=lugito Group=lugito WorkingDirectory=/srv/lubuntu-ci/repos/ci-tools -ExecStart=/srv/lubuntu-ci/repos/ci-tools/run-britney +ExecStart=/srv/lubuntu-ci/repos/ci-tools/fetch-indexes -c /srv/lubuntu-ci/repos/ci-tools/fetch-indexes.yaml diff --git a/systemd/pull-tooling.service b/systemd/pull-tooling.service new file mode 100644 index 0000000..82050b7 --- /dev/null +++ b/systemd/pull-tooling.service @@ -0,0 +1,11 @@ +[Unit] +Description=Pull and compile tooling +Wants=pull-tooling.timer +After=network.target + +[Service] +Type=simple +User=lugito +Group=lugito +WorkingDirectory=/srv/lubuntu-ci/repos/ci-tools/ +ExecStart=/srv/lubuntu-ci/repos/ci-tools/pull-and-compile-tooling diff --git a/systemd/pull-tooling.timer b/systemd/pull-tooling.timer new file mode 100644 index 0000000..6a39616 --- /dev/null +++ b/systemd/pull-tooling.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Pull and compile tooling timer + +[Timer] +OnBootSec=3min +OnUnitActiveSec=60min +Persistent=true + +[Install] +WantedBy=timers.target