parent
23894aede2
commit
706eb25581
@ -1 +1,3 @@
|
|||||||
__pycache__
|
__pycache__
|
||||||
|
fetch-indexes
|
||||||
|
fetch-indexes-cpp/build
|
||||||
|
|||||||
@ -0,0 +1,28 @@
|
|||||||
|
MAIN_ARCHIVE: "http://us.archive.ubuntu.com/ubuntu/dists/"
|
||||||
|
PORTS_ARCHIVE: "http://us.ports.ubuntu.com/ubuntu-ports/dists/"
|
||||||
|
LP_TEAM: "lubuntu-ci"
|
||||||
|
SOURCE_PPA: "unstable-ci-proposed"
|
||||||
|
DEST_PPA: "unstable-ci"
|
||||||
|
|
||||||
|
ARCHES:
|
||||||
|
- "amd64"
|
||||||
|
PORTS_ARCHES:
|
||||||
|
- "arm64"
|
||||||
|
- "armhf"
|
||||||
|
- "ppc64el"
|
||||||
|
- "riscv64"
|
||||||
|
- "s390x"
|
||||||
|
|
||||||
|
BRITNEY_CACHE: "cache/"
|
||||||
|
BRITNEY_DATADIR: "data/"
|
||||||
|
BRITNEY_OUTDIR: "output/"
|
||||||
|
BRITNEY_HINTDIR: "../hints-ubuntu/"
|
||||||
|
BRITNEY_LOC: "/srv/lubuntu-ci/repos/britney2-ubuntu/britney.py"
|
||||||
|
|
||||||
|
LOG_DIR: "/srv/lubuntu-ci/output/logs/britney"
|
||||||
|
MAX_LOG_AGE_DAYS: 1
|
||||||
|
|
||||||
|
RELEASES:
|
||||||
|
- "plucky"
|
||||||
|
- "oracular"
|
||||||
|
- "noble"
|
||||||
@ -1,161 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# download current package indexes to data/<series>{,-proposed}/ for running
|
|
||||||
# britney against a PPA. The PPA will play the role of "-proposed" (i. e.
|
|
||||||
# "unstable" in britney terms, containing the updated packages to test), the
|
|
||||||
# Ubuntu archive has the "-release" part (i. e. "testing" in britney terms, in
|
|
||||||
# which the -proposed packages are being landed).
|
|
||||||
#
|
|
||||||
# Copyright (C) 2019-2024 Simon Quigley <tsimonq2@ubuntu.com>
|
|
||||||
# Copyright (C) Canonical Ltd
|
|
||||||
# Author: Martin Pitt <martin.pitt@ubuntu.com>
|
|
||||||
# Author: Robert Bruce Park <robert.park@canonical.com>
|
|
||||||
# This program is free software: you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU General Public License as published by
|
|
||||||
# the Free Software Foundation, either version 3 of the License, or
|
|
||||||
# (at your option) any later version.
|
|
||||||
#
|
|
||||||
# This program is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
# GNU General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License
|
|
||||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
set -u
|
|
||||||
|
|
||||||
./pending-packages $RELEASE || exit 0
|
|
||||||
|
|
||||||
export MAIN_ARCHIVE="http://us.archive.ubuntu.com/ubuntu/dists/"
|
|
||||||
export PORTS_ARCHIVE="http://us.ports.ubuntu.com/ubuntu-ports/dists/"
|
|
||||||
export LP_TEAM="lubuntu-ci"
|
|
||||||
export SOURCE_PPA="unstable-ci-proposed"
|
|
||||||
export DEST_PPA="unstable-ci"
|
|
||||||
export SOURCE_PPA_URL="https://ppa.launchpadcontent.net/$LP_TEAM/$SOURCE_PPA/ubuntu/dists/$RELEASE/main";
|
|
||||||
export DEST_PPA_URL="https://ppa.launchpadcontent.net/$LP_TEAM/$DEST_PPA/ubuntu/dists/$RELEASE/main";
|
|
||||||
export ARCHES="amd64"
|
|
||||||
export PORTS_ARCHES="arm64 armhf ppc64el riscv64 s390x"
|
|
||||||
export BRITNEY_CACHE="cache/"
|
|
||||||
export BRITNEY_DATADIR="data/"
|
|
||||||
export BRITNEY_OUTDIR="output/"
|
|
||||||
export BRITNEY_HINTDIR="../hints-ubuntu/"
|
|
||||||
export BRITNEY_LOC="/srv/lubuntu-ci/repos/britney2-ubuntu/britney.py"
|
|
||||||
export BRITNEY_TIMESTAMP=$(date +"%Y-%m-%d_%H:%M:%S")
|
|
||||||
|
|
||||||
echo "Release: $RELEASE";
|
|
||||||
echo "Timestamp: $BRITNEY_TIMESTAMP"
|
|
||||||
|
|
||||||
# Download files in parallel in background, only if there is an update
|
|
||||||
refresh() {
|
|
||||||
DIR=$BRITNEY_CACHE/$pocket/$(echo $1 | rev | cut --delimiter=/ --fields=2,3 | rev)
|
|
||||||
mkdir --parents $DIR
|
|
||||||
touch --no-create $BRITNEY_CACHE $BRITNEY_CACHE/$pocket "$(dirname $DIR)" $DIR # Timestamp thwarts expire.sh
|
|
||||||
wget --directory-prefix $DIR --timestamping $1 --append-output $DIR/$$-wget-log --no-verbose &
|
|
||||||
}
|
|
||||||
|
|
||||||
echo 'Refreshing package indexes...'
|
|
||||||
|
|
||||||
for pocket in $RELEASE $RELEASE-updates; do
|
|
||||||
for component in main restricted universe multiverse; do
|
|
||||||
for arch in $ARCHES; do
|
|
||||||
refresh $MAIN_ARCHIVE/$pocket/$component/binary-$arch/Packages.gz
|
|
||||||
done
|
|
||||||
for arch in $PORTS_ARCHES; do
|
|
||||||
refresh $PORTS_ARCHIVE/$pocket/$component/binary-$arch/Packages.gz
|
|
||||||
done
|
|
||||||
refresh $MAIN_ARCHIVE/$pocket/$component/source/Sources.gz
|
|
||||||
done
|
|
||||||
done
|
|
||||||
|
|
||||||
# Treat the destination PPA as just another pocket
|
|
||||||
for pocket in $RELEASE-ppa-proposed; do
|
|
||||||
for arch in $ARCHES $PORTS_ARCHES; do
|
|
||||||
refresh $DEST_PPA_URL/source/Sources.gz
|
|
||||||
refresh $DEST_PPA_URL/binary-$arch/Packages.gz
|
|
||||||
done
|
|
||||||
done
|
|
||||||
|
|
||||||
# Get the source PPA
|
|
||||||
pocket=$SOURCE_PPA-$RELEASE
|
|
||||||
for arch in $ARCHES $PORTS_ARCHES; do
|
|
||||||
refresh $SOURCE_PPA_URL/binary-$arch/Packages.gz
|
|
||||||
done
|
|
||||||
refresh $SOURCE_PPA_URL/source/Sources.gz
|
|
||||||
|
|
||||||
wait # for wgets to finish
|
|
||||||
|
|
||||||
find $BRITNEY_DATADIR -name "$$-wget-log*" -exec cat '{}' \; -delete 1>&2
|
|
||||||
|
|
||||||
echo 'Building britney indexes...'
|
|
||||||
|
|
||||||
mkdir --parents "$BRITNEY_OUTDIR/$BRITNEY_TIMESTAMP/"
|
|
||||||
|
|
||||||
# "Unstable" is SOURCE_PPA
|
|
||||||
DEST=$BRITNEY_DATADIR/$RELEASE-proposed
|
|
||||||
mkdir --parents $DEST
|
|
||||||
mkdir -pv $BRITNEY_DATADIR/$RELEASE-proposed/state/
|
|
||||||
touch $BRITNEY_DATADIR/$RELEASE-proposed/state/age-policy-dates
|
|
||||||
touch --no-create $DEST
|
|
||||||
ln --verbose --symbolic --force --no-dereference $BRITNEY_HINTDIR $DEST/Hints
|
|
||||||
zcat $BRITNEY_CACHE/$SOURCE_PPA-$RELEASE/*/source/Sources.gz > $DEST/Sources
|
|
||||||
for arch in $ARCHES $PORTS_ARCHES; do
|
|
||||||
zcat $BRITNEY_CACHE/$SOURCE_PPA-$RELEASE/*/binary-$arch/Packages.gz > $DEST/Packages_${arch}
|
|
||||||
done
|
|
||||||
touch $DEST/Blocks $DEST/Dates
|
|
||||||
|
|
||||||
# "Testing" is a combination of the archive and DEST_PPA
|
|
||||||
DEST=$BRITNEY_DATADIR/$RELEASE
|
|
||||||
mkdir --parents $DEST
|
|
||||||
mkdir -pv $BRITNEY_DATADIR/$RELEASE/state/
|
|
||||||
touch $BRITNEY_DATADIR/$RELEASE/state/age-policy-dates
|
|
||||||
touch --no-create $DEST
|
|
||||||
ln --verbose --symbolic --force --no-dereference $BRITNEY_HINTDIR $DEST/Hints
|
|
||||||
zcat $BRITNEY_CACHE/$RELEASE*/*/source/Sources.gz > $DEST/Sources
|
|
||||||
sed -i "s/Section: universe\//Section: /g" $DEST/Sources
|
|
||||||
for arch in $ARCHES $PORTS_ARCHES; do
|
|
||||||
zcat $BRITNEY_CACHE/$RELEASE*/*/binary-$arch/Packages.gz > $DEST/Packages_${arch}
|
|
||||||
sed -i "s/Section: universe\//Section: /g" $DEST/Packages_${arch}
|
|
||||||
done
|
|
||||||
touch $DEST/Blocks
|
|
||||||
touch "$BRITNEY_DATADIR/$SOURCE_PPA-$RELEASE/Dates"
|
|
||||||
|
|
||||||
# Create config file atomically.
|
|
||||||
CONFIG="britney.conf"
|
|
||||||
cp $CONFIG $CONFIG.bak
|
|
||||||
envsubst < "$CONFIG.bak" > "$CONFIG"
|
|
||||||
rm $CONFIG.bak
|
|
||||||
|
|
||||||
echo 'Running britney...'
|
|
||||||
$BRITNEY_LOC -v --config "$CONFIG" --series $RELEASE
|
|
||||||
|
|
||||||
echo 'Syncing output to frontend...'
|
|
||||||
rmdir output/;
|
|
||||||
rsync -da output/ ../../output/britney
|
|
||||||
|
|
||||||
echo "$0 done."
|
|
||||||
|
|
||||||
echo "Moving packages..."
|
|
||||||
|
|
||||||
egrep -v '^#' output/$RELEASE/HeidiResultDelta > candidates || echo "No candidates found.";
|
|
||||||
|
|
||||||
while read -r -a package; do
|
|
||||||
# This only acts on sources; binaries require manual cleanup
|
|
||||||
if [ ${#package[@]} = 2 ]; then
|
|
||||||
COPY="../ubuntu-archive-tools/copy-package"
|
|
||||||
REMOVE="../ubuntu-archive-tools/remove-package"
|
|
||||||
if echo ${package[0]} | egrep -q "^-"; then
|
|
||||||
PACKAGE=$(echo ${package[0]} | sed 's/-//')
|
|
||||||
echo "Demoting $PACKAGE..."
|
|
||||||
$COPY -y -b -s $RELEASE --from "ppa:$LP_TEAM/ubuntu/$DEST_PPA" --to "ppa:$LP_TEAM/ubuntu/$SOURCE_PPA" --version "${package[1]}" "$PACKAGE";
|
|
||||||
$REMOVE -y -s $RELEASE --archive "ppa:$LP_TEAM/ubuntu/$DEST_PPA" --version "${package[1]}" --removal-comment="demoted to proposed" "$PACKAGE";
|
|
||||||
else
|
|
||||||
echo "Migrating ${package[0]}..."
|
|
||||||
$COPY -y -b -s $RELEASE --from "ppa:$LP_TEAM/ubuntu/$SOURCE_PPA" --to "ppa:$LP_TEAM/ubuntu/$DEST_PPA" --version "${package[1]}" "${package[0]}";
|
|
||||||
$REMOVE -y -s $RELEASE --archive "ppa:$LP_TEAM/ubuntu/$SOURCE_PPA" --version "${package[1]}" --removal-comment="moved to release" "${package[0]}";
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done < candidates;
|
|
||||||
rm candidates;
|
|
||||||
|
|
||||||
echo "Run the grim reaper..."
|
|
||||||
./grim-reaper $RELEASE
|
|
||||||
@ -0,0 +1,17 @@
|
|||||||
|
cmake_minimum_required(VERSION 3.10)
|
||||||
|
project(fetch-indexes)
|
||||||
|
|
||||||
|
set(CMAKE_CXX_STANDARD 23)
|
||||||
|
set(CMAKE_CXX_STANDARD_REQUIRED True)
|
||||||
|
|
||||||
|
find_package(CURL REQUIRED)
|
||||||
|
find_package(ZLIB REQUIRED)
|
||||||
|
find_package(YAML-CPP REQUIRED)
|
||||||
|
|
||||||
|
include_directories(${CURL_INCLUDE_DIRS})
|
||||||
|
include_directories(${ZLIB_INCLUDE_DIRS})
|
||||||
|
include_directories(${YAML_CPP_INCLUDE_DIR})
|
||||||
|
|
||||||
|
add_executable(fetch-indexes main.cpp utilities.cpp)
|
||||||
|
|
||||||
|
target_link_libraries(fetch-indexes ${CURL_LIBRARIES} ${ZLIB_LIBRARIES} yaml-cpp)
|
||||||
@ -0,0 +1,482 @@
|
|||||||
|
// Copyright (C) 2024 Simon Quigley <tsimonq2@ubuntu.com>
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
#include <iostream>
|
||||||
|
#include <fstream>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
#include <map>
|
||||||
|
#include <filesystem>
|
||||||
|
#include <thread>
|
||||||
|
#include <mutex>
|
||||||
|
#include <regex>
|
||||||
|
#include <ctime>
|
||||||
|
#include <chrono>
|
||||||
|
#include <cstdlib>
|
||||||
|
#include <cstring>
|
||||||
|
#include <cstdio>
|
||||||
|
#include <unistd.h>
|
||||||
|
#include <getopt.h>
|
||||||
|
#include <yaml-cpp/yaml.h>
|
||||||
|
|
||||||
|
#include "utilities.h"
|
||||||
|
|
||||||
|
namespace fs = std::filesystem;
|
||||||
|
|
||||||
|
// Function prototypes
|
||||||
|
void printHelp(const char* programName);
|
||||||
|
void processRelease(const std::string& release, const YAML::Node& config);
|
||||||
|
void refresh(const std::string& url, const std::string& pocket, const std::string& britneyCache, std::mutex& logMutex);
|
||||||
|
|
||||||
|
int main(int argc, char* argv[]) {
|
||||||
|
std::string configFilePath = "config.yaml";
|
||||||
|
|
||||||
|
// Command-line argument parsing
|
||||||
|
int opt;
|
||||||
|
bool showHelp = false;
|
||||||
|
|
||||||
|
static struct option long_options[] = {
|
||||||
|
{"config", required_argument, 0, 'c'},
|
||||||
|
{"help", no_argument, 0, 'h'},
|
||||||
|
{0, 0, 0, 0 }
|
||||||
|
};
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
int option_index = 0;
|
||||||
|
opt = getopt_long(argc, argv, "c:h", long_options, &option_index);
|
||||||
|
|
||||||
|
if (opt == -1)
|
||||||
|
break;
|
||||||
|
|
||||||
|
switch (opt) {
|
||||||
|
case 'c':
|
||||||
|
configFilePath = optarg;
|
||||||
|
break;
|
||||||
|
case 'h':
|
||||||
|
showHelp = true;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
// Unknown option
|
||||||
|
showHelp = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (showHelp) {
|
||||||
|
printHelp(argv[0]);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load configuration from YAML file
|
||||||
|
YAML::Node config;
|
||||||
|
try {
|
||||||
|
config = YAML::LoadFile(configFilePath);
|
||||||
|
} catch (const YAML::BadFile& e) {
|
||||||
|
std::cerr << "Error: Unable to open config file: " << configFilePath << std::endl;
|
||||||
|
return 1;
|
||||||
|
} catch (const YAML::ParserException& e) {
|
||||||
|
std::cerr << "Error: Failed to parse config file: " << e.what() << std::endl;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure LOG_DIR exists
|
||||||
|
std::string LOG_DIR = config["LOG_DIR"].as<std::string>();
|
||||||
|
fs::create_directories(LOG_DIR);
|
||||||
|
|
||||||
|
// Log rotation: Remove logs older than MAX_LOG_AGE_DAYS
|
||||||
|
int maxLogAgeDays = config["MAX_LOG_AGE_DAYS"].as<int>();
|
||||||
|
auto now = fs::file_time_type::clock::now(); // Use the same clock as file_time_type
|
||||||
|
|
||||||
|
for (const auto& entry : fs::directory_iterator(LOG_DIR)) {
|
||||||
|
if (entry.is_regular_file()) {
|
||||||
|
auto ftime = fs::last_write_time(entry.path());
|
||||||
|
auto age = std::chrono::duration_cast<std::chrono::hours>(now - ftime).count() / 24;
|
||||||
|
if (age > maxLogAgeDays) {
|
||||||
|
fs::remove(entry.path());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the list of releases
|
||||||
|
std::vector<std::string> releases = config["RELEASES"].as<std::vector<std::string>>();
|
||||||
|
|
||||||
|
// Process each release
|
||||||
|
for (const auto& release : releases) {
|
||||||
|
// Log file named by current UTC time (YYYYMMDDTHH:MM:SS) and release
|
||||||
|
std::time_t now_c = std::time(nullptr);
|
||||||
|
char timestamp[20];
|
||||||
|
std::strftime(timestamp, sizeof(timestamp), "%Y%m%dT%H:%M:%S", std::gmtime(&now_c));
|
||||||
|
std::string logFileName = LOG_DIR + "/" + timestamp + "_" + release + ".log";
|
||||||
|
|
||||||
|
// Open log file
|
||||||
|
std::ofstream logFile(logFileName, std::ios::app);
|
||||||
|
if (!logFile.is_open()) {
|
||||||
|
std::cerr << "Error: Unable to open log file: " << logFileName << std::endl;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Redirect stdout and stderr to log file
|
||||||
|
std::streambuf* coutBuf = std::cout.rdbuf();
|
||||||
|
std::streambuf* cerrBuf = std::cerr.rdbuf();
|
||||||
|
std::cout.rdbuf(logFile.rdbuf());
|
||||||
|
std::cerr.rdbuf(logFile.rdbuf());
|
||||||
|
|
||||||
|
// Log the start time
|
||||||
|
char startTime[20];
|
||||||
|
std::strftime(startTime, sizeof(startTime), "%Y-%m-%d %H:%M:%S", std::gmtime(&now_c));
|
||||||
|
std::cout << startTime << " - Running Britney for " << release << std::endl;
|
||||||
|
|
||||||
|
// Process the release
|
||||||
|
processRelease(release, config);
|
||||||
|
|
||||||
|
// Restore stdout and stderr
|
||||||
|
std::cout.rdbuf(coutBuf);
|
||||||
|
std::cerr.rdbuf(cerrBuf);
|
||||||
|
|
||||||
|
// Close log file
|
||||||
|
logFile.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void processRelease(const std::string& RELEASE, const YAML::Node& config) {
|
||||||
|
// Extract configuration variables
|
||||||
|
std::string MAIN_ARCHIVE = config["MAIN_ARCHIVE"].as<std::string>();
|
||||||
|
std::string PORTS_ARCHIVE = config["PORTS_ARCHIVE"].as<std::string>();
|
||||||
|
std::string LP_TEAM = config["LP_TEAM"].as<std::string>();
|
||||||
|
std::string SOURCE_PPA = config["SOURCE_PPA"].as<std::string>();
|
||||||
|
std::string DEST_PPA = config["DEST_PPA"].as<std::string>();
|
||||||
|
std::string BRITNEY_CACHE = config["BRITNEY_CACHE"].as<std::string>();
|
||||||
|
std::string BRITNEY_DATADIR = config["BRITNEY_DATADIR"].as<std::string>();
|
||||||
|
std::string BRITNEY_OUTDIR = config["BRITNEY_OUTDIR"].as<std::string>();
|
||||||
|
std::string BRITNEY_HINTDIR = config["BRITNEY_HINTDIR"].as<std::string>();
|
||||||
|
std::string BRITNEY_LOC = config["BRITNEY_LOC"].as<std::string>();
|
||||||
|
|
||||||
|
std::vector<std::string> ARCHES = config["ARCHES"].as<std::vector<std::string>>();
|
||||||
|
std::vector<std::string> PORTS_ARCHES = config["PORTS_ARCHES"].as<std::vector<std::string>>();
|
||||||
|
|
||||||
|
std::string SOURCE_PPA_URL = "https://ppa.launchpadcontent.net/" + LP_TEAM + "/" + SOURCE_PPA + "/ubuntu/dists/" + RELEASE + "/main";
|
||||||
|
std::string DEST_PPA_URL = "https://ppa.launchpadcontent.net/" + LP_TEAM + "/" + DEST_PPA + "/ubuntu/dists/" + RELEASE + "/main";
|
||||||
|
|
||||||
|
// Get current timestamp
|
||||||
|
std::time_t now_c = std::time(nullptr);
|
||||||
|
char timestamp[20];
|
||||||
|
std::strftime(timestamp, sizeof(timestamp), "%Y-%m-%d_%H:%M:%S", std::localtime(&now_c));
|
||||||
|
std::string BRITNEY_TIMESTAMP(timestamp);
|
||||||
|
|
||||||
|
std::cout << "Release: " << RELEASE << std::endl;
|
||||||
|
std::cout << "Timestamp: " << BRITNEY_TIMESTAMP << std::endl;
|
||||||
|
|
||||||
|
// Execute pending-packages script
|
||||||
|
std::string pendingCmd = "./pending-packages " + RELEASE;
|
||||||
|
int pendingResult = std::system(pendingCmd.c_str());
|
||||||
|
if (pendingResult != 0) {
|
||||||
|
std::cerr << "Error: pending-packages script failed for release " << RELEASE << std::endl;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cout << "Refreshing package indexes..." << std::endl;
|
||||||
|
|
||||||
|
std::vector<std::thread> threads;
|
||||||
|
std::mutex logMutex;
|
||||||
|
|
||||||
|
// Refresh package indexes
|
||||||
|
std::vector<std::string> pockets = {RELEASE, RELEASE + "-updates"};
|
||||||
|
std::vector<std::string> components = {"main", "restricted", "universe", "multiverse"};
|
||||||
|
|
||||||
|
// Loop over pockets, components, architectures to refresh package indexes
|
||||||
|
for (const auto& pocket : pockets) {
|
||||||
|
for (const auto& component : components) {
|
||||||
|
for (const auto& arch : ARCHES) {
|
||||||
|
std::string url = MAIN_ARCHIVE + pocket + "/" + component + "/binary-" + arch + "/Packages.gz";
|
||||||
|
threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
|
||||||
|
}
|
||||||
|
for (const auto& arch : PORTS_ARCHES) {
|
||||||
|
std::string url = PORTS_ARCHIVE + pocket + "/" + component + "/binary-" + arch + "/Packages.gz";
|
||||||
|
threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
|
||||||
|
}
|
||||||
|
std::string url = MAIN_ARCHIVE + pocket + "/" + component + "/source/Sources.gz";
|
||||||
|
threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Treat the destination PPA as just another pocket
|
||||||
|
std::string pocket = RELEASE + "-ppa-proposed";
|
||||||
|
for (const auto& arch : ARCHES) {
|
||||||
|
std::string url = DEST_PPA_URL + "/binary-" + arch + "/Packages.gz";
|
||||||
|
threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
|
||||||
|
}
|
||||||
|
for (const auto& arch : PORTS_ARCHES) {
|
||||||
|
std::string url = DEST_PPA_URL + "/binary-" + arch + "/Packages.gz";
|
||||||
|
threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
|
||||||
|
}
|
||||||
|
{
|
||||||
|
std::string url = DEST_PPA_URL + "/source/Sources.gz";
|
||||||
|
threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the source PPA
|
||||||
|
pocket = SOURCE_PPA + "-" + RELEASE;
|
||||||
|
for (const auto& arch : ARCHES) {
|
||||||
|
std::string url = SOURCE_PPA_URL + "/binary-" + arch + "/Packages.gz";
|
||||||
|
threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
|
||||||
|
}
|
||||||
|
for (const auto& arch : PORTS_ARCHES) {
|
||||||
|
std::string url = SOURCE_PPA_URL + "/binary-" + arch + "/Packages.gz";
|
||||||
|
threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
|
||||||
|
}
|
||||||
|
{
|
||||||
|
std::string url = SOURCE_PPA_URL + "/source/Sources.gz";
|
||||||
|
threads.emplace_back(refresh, url, pocket, BRITNEY_CACHE, std::ref(logMutex));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for all threads to finish
|
||||||
|
for (auto& th : threads) {
|
||||||
|
th.join();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process logs and delete them
|
||||||
|
pid_t pid = getpid();
|
||||||
|
std::string logPattern = std::to_string(pid) + "-wget-log";
|
||||||
|
|
||||||
|
for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE)) {
|
||||||
|
if (p.is_regular_file()) {
|
||||||
|
std::string filename = p.path().filename().string();
|
||||||
|
if (filename.find(logPattern) != std::string::npos) {
|
||||||
|
// Output log content to stderr
|
||||||
|
std::ifstream logFile(p.path());
|
||||||
|
std::cerr << logFile.rdbuf();
|
||||||
|
logFile.close();
|
||||||
|
fs::remove(p.path());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cout << "Building britney indexes..." << std::endl;
|
||||||
|
|
||||||
|
// Create output directory
|
||||||
|
fs::create_directories(fs::path(BRITNEY_OUTDIR) / BRITNEY_TIMESTAMP);
|
||||||
|
|
||||||
|
// "Unstable" is SOURCE_PPA
|
||||||
|
std::string DEST = BRITNEY_DATADIR + RELEASE + "-proposed";
|
||||||
|
fs::create_directories(DEST);
|
||||||
|
fs::create_directories(fs::path(BRITNEY_DATADIR) / (RELEASE + "-proposed") / "state");
|
||||||
|
writeFile(fs::path(BRITNEY_DATADIR) / (RELEASE + "-proposed") / "state" / "age-policy-dates", "");
|
||||||
|
|
||||||
|
// Create symlink for Hints
|
||||||
|
fs::remove(fs::path(DEST) / "Hints");
|
||||||
|
fs::create_symlink(BRITNEY_HINTDIR, fs::path(DEST) / "Hints");
|
||||||
|
|
||||||
|
// Concatenate Sources.gz files for SOURCE_PPA
|
||||||
|
std::string sourcesContent;
|
||||||
|
for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE + SOURCE_PPA + "-" + RELEASE)) {
|
||||||
|
if (p.path().filename() == "Sources.gz") {
|
||||||
|
sourcesContent += decompressGzip(p.path());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeFile(fs::path(DEST) / "Sources", sourcesContent);
|
||||||
|
|
||||||
|
// Concatenate Packages.gz files for SOURCE_PPA
|
||||||
|
for (const auto& arch : ARCHES) {
|
||||||
|
std::string packagesContent;
|
||||||
|
for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE + SOURCE_PPA + "-" + RELEASE)) {
|
||||||
|
if (p.path().filename() == "Packages.gz" && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) {
|
||||||
|
packagesContent += decompressGzip(p.path());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeFile(fs::path(DEST) / ("Packages_" + arch), packagesContent);
|
||||||
|
}
|
||||||
|
for (const auto& arch : PORTS_ARCHES) {
|
||||||
|
std::string packagesContent;
|
||||||
|
for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE + SOURCE_PPA + "-" + RELEASE)) {
|
||||||
|
if (p.path().filename() == "Packages.gz" && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) {
|
||||||
|
packagesContent += decompressGzip(p.path());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeFile(fs::path(DEST) / ("Packages_" + arch), packagesContent);
|
||||||
|
}
|
||||||
|
|
||||||
|
writeFile(fs::path(DEST) / "Blocks", "");
|
||||||
|
writeFile(fs::path(DEST) / "Dates", "");
|
||||||
|
|
||||||
|
// Similar steps for "Testing"
|
||||||
|
DEST = BRITNEY_DATADIR + RELEASE;
|
||||||
|
fs::create_directories(DEST);
|
||||||
|
fs::create_directories(fs::path(BRITNEY_DATADIR) / RELEASE / "state");
|
||||||
|
writeFile(fs::path(BRITNEY_DATADIR) / RELEASE / "state" / "age-policy-dates", "");
|
||||||
|
|
||||||
|
fs::remove(fs::path(DEST) / "Hints");
|
||||||
|
fs::create_symlink(BRITNEY_HINTDIR, fs::path(DEST) / "Hints");
|
||||||
|
|
||||||
|
// Concatenate Sources.gz files for RELEASE
|
||||||
|
sourcesContent.clear();
|
||||||
|
for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE)) {
|
||||||
|
if (p.path().filename() == "Sources.gz" && p.path().string().find(RELEASE) != std::string::npos) {
|
||||||
|
sourcesContent += decompressGzip(p.path());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeFile(fs::path(DEST) / "Sources", sourcesContent);
|
||||||
|
// Replace "Section: universe/" with "Section: "
|
||||||
|
regexReplaceInFile(fs::path(DEST) / "Sources", "Section: universe/", "Section: ");
|
||||||
|
|
||||||
|
// Concatenate Packages.gz files for RELEASE
|
||||||
|
for (const auto& arch : ARCHES) {
|
||||||
|
std::string packagesContent;
|
||||||
|
for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE)) {
|
||||||
|
if (p.path().filename() == "Packages.gz" && p.path().string().find(RELEASE) != std::string::npos && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) {
|
||||||
|
packagesContent += decompressGzip(p.path());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fs::path packagesFilePath = fs::path(DEST) / ("Packages_" + arch);
|
||||||
|
writeFile(packagesFilePath, packagesContent);
|
||||||
|
// Replace "Section: universe/" with "Section: "
|
||||||
|
regexReplaceInFile(packagesFilePath, "Section: universe/", "Section: ");
|
||||||
|
}
|
||||||
|
for (const auto& arch : PORTS_ARCHES) {
|
||||||
|
std::string packagesContent;
|
||||||
|
for (auto& p : fs::recursive_directory_iterator(BRITNEY_CACHE)) {
|
||||||
|
if (p.path().filename() == "Packages.gz" && p.path().string().find(RELEASE) != std::string::npos && p.path().parent_path().string().find("binary-" + arch) != std::string::npos) {
|
||||||
|
packagesContent += decompressGzip(p.path());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fs::path packagesFilePath = fs::path(DEST) / ("Packages_" + arch);
|
||||||
|
writeFile(packagesFilePath, packagesContent);
|
||||||
|
// Replace "Section: universe/" with "Section: "
|
||||||
|
regexReplaceInFile(packagesFilePath, "Section: universe/", "Section: ");
|
||||||
|
}
|
||||||
|
|
||||||
|
writeFile(fs::path(DEST) / "Blocks", "");
|
||||||
|
writeFile(fs::path(BRITNEY_DATADIR) / (SOURCE_PPA + "-" + RELEASE) / "Dates", "");
|
||||||
|
|
||||||
|
// Create config file atomically
|
||||||
|
std::string configContent = readFile("britney.conf");
|
||||||
|
// Replace variables in configContent using configuration variables
|
||||||
|
configContent = std::regex_replace(configContent, std::regex("\\$\\{RELEASE\\}"), RELEASE);
|
||||||
|
configContent = std::regex_replace(configContent, std::regex("\\$\\{BRITNEY_DATADIR\\}"), BRITNEY_DATADIR);
|
||||||
|
writeFile("britney.conf", configContent);
|
||||||
|
|
||||||
|
std::cout << "Running britney..." << std::endl;
|
||||||
|
|
||||||
|
// Run britney.py
|
||||||
|
std::string britneyCmd = BRITNEY_LOC + " -v --config britney.conf --series " + RELEASE;
|
||||||
|
int britneyResult = std::system(britneyCmd.c_str());
|
||||||
|
if (britneyResult != 0) {
|
||||||
|
std::cerr << "Error: Britney execution failed for release " << RELEASE << std::endl;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cout << "Syncing output to frontend..." << std::endl;
|
||||||
|
// Rsync command can be replaced with filesystem copy
|
||||||
|
fs::remove_all("output");
|
||||||
|
fs::copy("output", "../../output/britney", fs::copy_options::recursive | fs::copy_options::overwrite_existing);
|
||||||
|
|
||||||
|
std::cout << "Moving packages..." << std::endl;
|
||||||
|
|
||||||
|
// Read candidates from HeidiResultDelta
|
||||||
|
std::ifstream heidiFile("output/" + RELEASE + "/HeidiResultDelta");
|
||||||
|
if (!heidiFile.is_open()) {
|
||||||
|
std::cout << "No candidates found for release " << RELEASE << "." << std::endl;
|
||||||
|
} else {
|
||||||
|
std::ofstream candidatesFile("candidates");
|
||||||
|
std::string line;
|
||||||
|
while (std::getline(heidiFile, line)) {
|
||||||
|
if (line.empty() || line[0] == '#') continue;
|
||||||
|
candidatesFile << line << std::endl;
|
||||||
|
}
|
||||||
|
heidiFile.close();
|
||||||
|
candidatesFile.close();
|
||||||
|
|
||||||
|
// Process candidates
|
||||||
|
std::ifstream candidates("candidates");
|
||||||
|
while (std::getline(candidates, line)) {
|
||||||
|
std::istringstream iss(line);
|
||||||
|
std::vector<std::string> packageInfo;
|
||||||
|
std::string word;
|
||||||
|
while (iss >> word) {
|
||||||
|
packageInfo.push_back(word);
|
||||||
|
}
|
||||||
|
if (packageInfo.size() == 2) {
|
||||||
|
std::string COPY = "../ubuntu-archive-tools/copy-package";
|
||||||
|
std::string REMOVE = "../ubuntu-archive-tools/remove-package";
|
||||||
|
if (packageInfo[0][0] == '-') {
|
||||||
|
std::string PACKAGE = packageInfo[0].substr(1);
|
||||||
|
std::cout << "Demoting " << PACKAGE << "..." << std::endl;
|
||||||
|
std::string copyCmd = COPY + " -y -b -s " + RELEASE + " --from ppa:" + LP_TEAM + "/ubuntu/" + DEST_PPA +
|
||||||
|
" --to ppa:" + LP_TEAM + "/ubuntu/" + SOURCE_PPA + " --version " + packageInfo[1] + " " + PACKAGE;
|
||||||
|
std::string removeCmd = REMOVE + " -y -s " + RELEASE + " --archive ppa:" + LP_TEAM + "/ubuntu/" + DEST_PPA +
|
||||||
|
" --version " + packageInfo[1] + " --removal-comment=\"demoted to proposed\" " + PACKAGE;
|
||||||
|
std::system(copyCmd.c_str());
|
||||||
|
std::system(removeCmd.c_str());
|
||||||
|
} else {
|
||||||
|
std::cout << "Migrating " << packageInfo[0] << "..." << std::endl;
|
||||||
|
std::string copyCmd = COPY + " -y -b -s " + RELEASE + " --from ppa:" + LP_TEAM + "/ubuntu/" + SOURCE_PPA +
|
||||||
|
" --to ppa:" + LP_TEAM + "/ubuntu/" + DEST_PPA + " --version " + packageInfo[1] + " " + packageInfo[0];
|
||||||
|
std::string removeCmd = REMOVE + " -y -s " + RELEASE + " --archive ppa:" + LP_TEAM + "/ubuntu/" + SOURCE_PPA +
|
||||||
|
" --version " + packageInfo[1] + " --removal-comment=\"moved to release\" " + packageInfo[0];
|
||||||
|
std::system(copyCmd.c_str());
|
||||||
|
std::system(removeCmd.c_str());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
candidates.close();
|
||||||
|
fs::remove("candidates");
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cout << "Run the grim reaper..." << std::endl;
|
||||||
|
std::string grimCmd = "./grim-reaper " + RELEASE;
|
||||||
|
int grimResult = std::system(grimCmd.c_str());
|
||||||
|
if (grimResult != 0) {
|
||||||
|
std::cerr << "Error: Grim reaper execution failed for release " << RELEASE << std::endl;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::cout << "Done processing release " << RELEASE << "." << std::endl;
|
||||||
|
}
|
||||||
|
|
||||||
|
void printHelp(const char* programName) {
|
||||||
|
std::cout << "Usage: " << programName << " [options]\n";
|
||||||
|
std::cout << "Options:\n";
|
||||||
|
std::cout << " -c, --config <file> Specify path to config file (default: config.yaml)\n";
|
||||||
|
std::cout << " -h, --help Show this help message and exit\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Refresh package indexes by downloading files in parallel
|
||||||
|
void refresh(const std::string& url, const std::string& pocket, const std::string& britneyCache, std::mutex& logMutex) {
|
||||||
|
// Compute directory path based on the URL
|
||||||
|
fs::path urlPath(url);
|
||||||
|
std::string lastTwoComponents = urlPath.parent_path().parent_path().filename().string() + "/" + urlPath.parent_path().filename().string();
|
||||||
|
fs::path dir = fs::path(britneyCache) / pocket / lastTwoComponents;
|
||||||
|
|
||||||
|
// Create necessary directories
|
||||||
|
fs::create_directories(dir);
|
||||||
|
|
||||||
|
// Update timestamps to prevent expiration
|
||||||
|
auto now = fs::file_time_type::clock::now(); // Use the same clock
|
||||||
|
fs::last_write_time(britneyCache, now);
|
||||||
|
fs::last_write_time(fs::path(britneyCache) / pocket, now);
|
||||||
|
fs::last_write_time(dir.parent_path(), now);
|
||||||
|
fs::last_write_time(dir, now);
|
||||||
|
|
||||||
|
// Log file path (uses process ID)
|
||||||
|
pid_t pid = getpid();
|
||||||
|
fs::path logFilePath = dir / (std::to_string(pid) + "-wget-log");
|
||||||
|
|
||||||
|
// Output file path
|
||||||
|
fs::path outputPath = dir / urlPath.filename();
|
||||||
|
|
||||||
|
// Download the file
|
||||||
|
downloadFileWithTimestamping(url, outputPath, logFilePath, logMutex);
|
||||||
|
}
|
||||||
@ -0,0 +1,132 @@
|
|||||||
|
// Copyright (C) 2024 Simon Quigley <tsimonq2@ubuntu.com>
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
#include "utilities.h"
|
||||||
|
|
||||||
|
#include <fstream>
|
||||||
|
#include <iostream>
|
||||||
|
#include <filesystem>
|
||||||
|
#include <regex>
|
||||||
|
#include <zlib.h>
|
||||||
|
#include <curl/curl.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
|
||||||
|
namespace fs = std::filesystem;
|
||||||
|
|
||||||
|
// Function to read the entire content of a file into a string
|
||||||
|
std::string readFile(const fs::path& filePath) {
|
||||||
|
std::ifstream inFile(filePath, std::ios::binary);
|
||||||
|
if (inFile) {
|
||||||
|
return std::string((std::istreambuf_iterator<char>(inFile)),
|
||||||
|
std::istreambuf_iterator<char>());
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Function to write a string into a file
|
||||||
|
void writeFile(const fs::path& filePath, const std::string& content) {
|
||||||
|
std::ofstream outFile(filePath, std::ios::binary);
|
||||||
|
if (outFile) {
|
||||||
|
outFile << content;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Function to perform in-place regex replace on a file
|
||||||
|
void regexReplaceInFile(const fs::path& filePath, const std::string& pattern, const std::string& replace) {
|
||||||
|
std::string content = readFile(filePath);
|
||||||
|
content = std::regex_replace(content, std::regex(pattern), replace);
|
||||||
|
writeFile(filePath, content);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Function to decompress gzipped files
|
||||||
|
std::string decompressGzip(const fs::path& filePath) {
|
||||||
|
gzFile infile = gzopen(filePath.c_str(), "rb");
|
||||||
|
if (!infile) return "";
|
||||||
|
|
||||||
|
std::string decompressedData;
|
||||||
|
char buffer[8192];
|
||||||
|
int numRead = 0;
|
||||||
|
while ((numRead = gzread(infile, buffer, sizeof(buffer))) > 0) {
|
||||||
|
decompressedData.append(buffer, numRead);
|
||||||
|
}
|
||||||
|
gzclose(infile);
|
||||||
|
return decompressedData;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function for libcurl write callback
|
||||||
|
size_t write_data(void* ptr, size_t size, size_t nmemb, void* stream) {
|
||||||
|
FILE* out = static_cast<FILE*>(stream);
|
||||||
|
return fwrite(ptr, size, nmemb, out);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Function to download a file with timestamping using libcurl
|
||||||
|
void downloadFileWithTimestamping(const std::string& url, const fs::path& outputPath,
|
||||||
|
const fs::path& logFilePath, std::mutex& logMutex) {
|
||||||
|
CURL* curl;
|
||||||
|
CURLcode res;
|
||||||
|
FILE* fp;
|
||||||
|
curl = curl_easy_init();
|
||||||
|
if (curl) {
|
||||||
|
fs::path tempFilePath = outputPath.string() + ".tmp";
|
||||||
|
fp = fopen(tempFilePath.c_str(), "wb");
|
||||||
|
|
||||||
|
if (!fp) {
|
||||||
|
std::cerr << "Failed to open file: " << tempFilePath << std::endl;
|
||||||
|
curl_easy_cleanup(curl);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set curl options for downloading the file
|
||||||
|
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
||||||
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
|
||||||
|
|
||||||
|
// Timestamping: set If-Modified-Since header
|
||||||
|
struct stat file_info;
|
||||||
|
if (stat(outputPath.c_str(), &file_info) == 0) {
|
||||||
|
// Set the time condition to If-Modified-Since
|
||||||
|
curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_TIMEVALUE, file_info.st_mtime);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Perform the file download
|
||||||
|
res = curl_easy_perform(curl);
|
||||||
|
|
||||||
|
// Get the HTTP response code
|
||||||
|
long response_code = 0;
|
||||||
|
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
|
||||||
|
|
||||||
|
fclose(fp);
|
||||||
|
curl_easy_cleanup(curl);
|
||||||
|
|
||||||
|
// Log the result and handle the downloaded file
|
||||||
|
{
|
||||||
|
std::lock_guard<std::mutex> lock(logMutex);
|
||||||
|
std::ofstream logFile(logFilePath, std::ios::app);
|
||||||
|
if (res == CURLE_OK && (response_code == 200 || response_code == 201)) {
|
||||||
|
fs::rename(tempFilePath, outputPath);
|
||||||
|
logFile << "Downloaded: " << url << std::endl;
|
||||||
|
} else if (response_code == 304) {
|
||||||
|
fs::remove(tempFilePath);
|
||||||
|
logFile << "Not Modified: " << url << std::endl;
|
||||||
|
} else {
|
||||||
|
fs::remove(tempFilePath);
|
||||||
|
logFile << "Failed to download: " << url << std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
std::cerr << "Failed to initialize CURL." << std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,42 @@
|
|||||||
|
// Copyright (C) 2024 Simon Quigley <tsimonq2@ubuntu.com>
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
#ifndef UTILITIES_H
|
||||||
|
#define UTILITIES_H
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
#include <filesystem>
|
||||||
|
#include <mutex>
|
||||||
|
|
||||||
|
// Function to read the entire content of a file into a string
|
||||||
|
std::string readFile(const std::filesystem::path& filePath);
|
||||||
|
|
||||||
|
// Function to write a string into a file
|
||||||
|
void writeFile(const std::filesystem::path& filePath, const std::string& content);
|
||||||
|
|
||||||
|
// Function to perform in-place regex replace on a file
|
||||||
|
void regexReplaceInFile(const std::filesystem::path& filePath, const std::string& pattern, const std::string& replace);
|
||||||
|
|
||||||
|
// Function to decompress gzipped files
|
||||||
|
std::string decompressGzip(const std::filesystem::path& filePath);
|
||||||
|
|
||||||
|
// Function to download a file with timestamping using libcurl
|
||||||
|
void downloadFileWithTimestamping(const std::string& url, const std::filesystem::path& outputPath,
|
||||||
|
const std::filesystem::path& logFilePath, std::mutex& logMutex);
|
||||||
|
|
||||||
|
// Helper function for libcurl write callback
|
||||||
|
size_t write_data(void* ptr, size_t size, size_t nmemb, void* stream);
|
||||||
|
|
||||||
|
#endif // UTILITIES_H
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
git pull
|
||||||
|
|
||||||
|
(cd fetch-indexes-cpp &&
|
||||||
|
rm -rf build; mkdir build && (cd build &&
|
||||||
|
cmake .. && make -j$(nproc)) &&
|
||||||
|
mv build/fetch-indexes ../ && rm -rf build)
|
||||||
@ -1,37 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
#
|
|
||||||
# Copyright (C) 2024 Simon Quigley <tsimonq2@ubuntu.com>
|
|
||||||
#
|
|
||||||
# This program is free software: you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU General Public License as published by
|
|
||||||
# the Free Software Foundation, either version 3 of the License, or
|
|
||||||
# (at your option) any later version.
|
|
||||||
#
|
|
||||||
# This program is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
# GNU General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License
|
|
||||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
# Configuration
|
|
||||||
LOG_DIR="/srv/lubuntu-ci/output/logs/britney"
|
|
||||||
SCRIPT_PATH="/srv/lubuntu-ci/repos/ci-tools/fetch-indexes"
|
|
||||||
|
|
||||||
# Ensure the log directory exists
|
|
||||||
mkdir -p $LOG_DIR
|
|
||||||
|
|
||||||
# Log rotation: Remove logs older than MAX_LOG_AGE
|
|
||||||
find $LOG_DIR -type f -mtime +1 -exec rm -f {} \;
|
|
||||||
|
|
||||||
# Execute the fetch-indexes script for each release and log output
|
|
||||||
for release in plucky oracular noble; do
|
|
||||||
export RELEASE="$release"
|
|
||||||
|
|
||||||
# Log file named by current UTC time (HH-MM-SS)
|
|
||||||
LOG_FILE="$LOG_DIR/$(date -u +"%Y%m%dT%H:%M:%S")_${release}.log"
|
|
||||||
|
|
||||||
echo "$(date -u +"%Y-%m-%d %H:%M:%S") - Running Britney for $RELEASE" >> "$LOG_FILE"
|
|
||||||
"$SCRIPT_PATH" >> "$LOG_FILE" 2>&1
|
|
||||||
done
|
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Pull and compile tooling
|
||||||
|
Wants=pull-tooling.timer
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=lugito
|
||||||
|
Group=lugito
|
||||||
|
WorkingDirectory=/srv/lubuntu-ci/repos/ci-tools/
|
||||||
|
ExecStart=/srv/lubuntu-ci/repos/ci-tools/pull-and-compile-tooling
|
||||||
@ -0,0 +1,10 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Pull and compile tooling timer
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=3min
|
||||||
|
OnUnitActiveSec=60min
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
Loading…
Reference in new issue