Further build-packages cleanup
This commit is contained in:
parent
6d81551a5d
commit
9abde5de3f
@ -15,6 +15,7 @@
|
||||
|
||||
#include "common.h"
|
||||
#include "update-maintainer-lib.h"
|
||||
#include "utilities.h"
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
@ -98,16 +99,314 @@ static int worker_count = 5;
|
||||
static bool verbose = false;
|
||||
static std::ofstream log_file_stream;
|
||||
|
||||
static std::string get_current_utc_time() {
|
||||
// Function declarations
|
||||
static std::optional<std::string> run_lintian(const fs::path& source_path);
|
||||
static void log_all(const std::string &level, const std::string &msg, bool is_error=false);
|
||||
static void log_info(const std::string &msg);
|
||||
static void log_warning(const std::string &msg);
|
||||
static void log_error(const std::string &msg);
|
||||
static void log_verbose(const std::string &msg);
|
||||
static void print_help(const std::string &prog_name);
|
||||
static void run_command_silent_on_success(const std::vector<std::string> &cmd, const std::optional<fs::path> &cwd = std::nullopt);
|
||||
static void git_init_once();
|
||||
static void git_fetch_and_checkout(const fs::path &repo_path, const std::string &repo_url, const std::optional<std::string> &branch);
|
||||
static YAML::Node load_config(const fs::path &config_path);
|
||||
static void publish_lintian();
|
||||
static std::vector<std::string> get_exclusions(const fs::path &packaging);
|
||||
static void run_source_lintian(const std::string &name, const fs::path &source_path);
|
||||
static void dput_source(const Package &pkg);
|
||||
static void update_changelog(const fs::path &packaging_dir, const std::string &release, const std::string &version_with_epoch);
|
||||
static std::string build_package(const fs::path &packaging_dir, const std::map<std::string, std::string> &env_vars, bool large, const std::string &pkg_name);
|
||||
static void pull_package(Package &pkg, const YAML::Node &releases);
|
||||
static void build_package_stage(Package &pkg, const YAML::Node &releases);
|
||||
static void build_package_stage_wrapper(Package &pkg, const YAML::Node &releases);
|
||||
static void upload_package_stage(Package &pkg, bool skip_dput);
|
||||
static void run_lintian_stage(Package &pkg);
|
||||
static void clean_old_logs_local(const fs::path &log_dir); // Removed to avoid ambiguity
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
std::string prog_name = fs::path(argv[0]).filename().string();
|
||||
for(int i = 1; i < argc; i++) {
|
||||
std::string arg = argv[i];
|
||||
if(arg == "--help" || arg == "-h") {
|
||||
print_help(prog_name);
|
||||
return 0;
|
||||
}
|
||||
if(arg == "--verbose" || arg == "-v") {
|
||||
verbose = true;
|
||||
// Remove the verbose flag from argv
|
||||
for(int j = i; j < argc - 1; j++) {
|
||||
argv[j] = argv[j+1];
|
||||
}
|
||||
argc--;
|
||||
i--;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
log_info("Script started.");
|
||||
fs::create_directories(LOG_DIR);
|
||||
log_info("Ensured log directory exists: " + LOG_DIR);
|
||||
fs::create_directories(OUTPUT_DIR);
|
||||
log_info("Ensured output directory exists: " + OUTPUT_DIR);
|
||||
|
||||
auto now = std::time(nullptr);
|
||||
std::tm tm_utc;
|
||||
gmtime_r(&now, &tm_utc);
|
||||
char buf[20];
|
||||
std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tm_utc);
|
||||
return std::string(buf);
|
||||
std::tm tm_time;
|
||||
gmtime_r(&now, &tm_time);
|
||||
char buf_time[20];
|
||||
std::strftime(buf_time, sizeof(buf_time), "%Y%m%dT%H%M%S", &tm_time);
|
||||
std::string current_time = buf_time;
|
||||
|
||||
std::string uuid_part = current_time.substr(0,10);
|
||||
BASE_LINTIAN_DIR = BASE_OUTPUT_DIR + "/.lintian.tmp." + uuid_part;
|
||||
fs::create_directories(BASE_LINTIAN_DIR);
|
||||
log_info("Created Lintian temporary directory: " + BASE_LINTIAN_DIR);
|
||||
|
||||
fs::path log_file = fs::path(LOG_DIR) / (current_time + ".log");
|
||||
log_info("Opening log file: " + log_file.string());
|
||||
log_file_stream.open(log_file);
|
||||
if(!log_file_stream.is_open()) {
|
||||
std::cerr << "[ERROR] Unable to open log file.\n";
|
||||
return 1;
|
||||
}
|
||||
log_info("Log file opened successfully.");
|
||||
|
||||
bool skip_dput = false;
|
||||
bool skip_cleanup = false;
|
||||
std::string config_path;
|
||||
for(int i = 1; i < argc; i++) {
|
||||
std::string arg = argv[i];
|
||||
log_info("Processing argument: " + arg);
|
||||
if(arg == "--skip-dput") {
|
||||
skip_dput = true;
|
||||
log_info("Flag set: --skip-dput");
|
||||
} else if(arg == "--skip-cleanup") {
|
||||
skip_cleanup = true;
|
||||
log_info("Flag set: --skip-cleanup");
|
||||
} else if(arg.rfind("--urgency-level=", 0) == 0) {
|
||||
urgency_level_override = arg.substr(std::string("--urgency-level=").size());
|
||||
log_info("Urgency level overridden to: " + urgency_level_override);
|
||||
} else if(arg.rfind("--workers=", 0) == 0) {
|
||||
try {
|
||||
worker_count = std::stoi(arg.substr(std::string("--workers=").size()));
|
||||
if(worker_count < 1) worker_count = 1;
|
||||
log_info("Worker count set to: " + std::to_string(worker_count));
|
||||
} catch(const std::exception &e) {
|
||||
log_error("Invalid worker count provided. Using default value of 5.");
|
||||
worker_count = 5;
|
||||
}
|
||||
} else if(config_path.empty()) {
|
||||
config_path = arg;
|
||||
log_info("Config path set to: " + config_path);
|
||||
}
|
||||
}
|
||||
|
||||
if(config_path.empty()) {
|
||||
log_error("No config file specified.");
|
||||
print_help(prog_name);
|
||||
return 1;
|
||||
}
|
||||
|
||||
setenv("DEBFULLNAME", DEBFULLNAME.c_str(), 1);
|
||||
log_info("Set DEBFULLNAME to: " + DEBFULLNAME);
|
||||
setenv("DEBEMAIL", DEBEMAIL.c_str(), 1);
|
||||
log_info("Set DEBEMAIL to: " + DEBEMAIL);
|
||||
|
||||
YAML::Node config;
|
||||
try {
|
||||
config = load_config(config_path);
|
||||
} catch (std::exception &e) {
|
||||
log_error(std::string("Error loading config file: ") + e.what());
|
||||
return 1;
|
||||
}
|
||||
|
||||
auto packages_node = config["packages"];
|
||||
auto releases = config["releases"];
|
||||
log_info("Loaded " + std::to_string(packages_node.size()) + " packages and " + std::to_string(releases.size()) + " releases from config.");
|
||||
|
||||
// Populate the packages vector
|
||||
std::vector<Package> packages;
|
||||
for(auto pkg_node : packages_node) {
|
||||
Package pkg;
|
||||
pkg.name = pkg_node["name"] ? pkg_node["name"].as<std::string>() : "";
|
||||
pkg.upload_target = pkg_node["upload_target"] ? pkg_node["upload_target"].as<std::string>() : "ppa:lubuntu-ci/unstable-ci-proposed";
|
||||
pkg.upstream_url = pkg_node["upstream_url"] ? pkg_node["upstream_url"].as<std::string>() : ("https://github.com/lxqt/" + pkg.name + ".git");
|
||||
pkg.packaging_url = pkg_node["packaging_url"] ? pkg_node["packaging_url"].as<std::string>() : ("https://git.lubuntu.me/Lubuntu/" + pkg.name + "-packaging.git");
|
||||
if(pkg_node["packaging_branch"] && pkg_node["packaging_branch"].IsScalar()) {
|
||||
pkg.packaging_branch = pkg_node["packaging_branch"].as<std::string>();
|
||||
}
|
||||
pkg.large = pkg_node["large"] ? pkg_node["large"].as<bool>() : false;
|
||||
|
||||
if(pkg.name.empty()) {
|
||||
log_warning("Skipping package due to missing name.");
|
||||
continue;
|
||||
}
|
||||
packages.emplace_back(std::move(pkg));
|
||||
}
|
||||
log_info("Prepared " + std::to_string(packages.size()) + " packages for processing.");
|
||||
|
||||
fs::current_path(BASE_DIR);
|
||||
log_info("Set current working directory to BASE_DIR: " + BASE_DIR);
|
||||
|
||||
// Stage 1: Pull all packages in parallel
|
||||
log_info("Starting Stage 1: Pulling all packages.");
|
||||
std::vector<std::future<void>> pull_futures;
|
||||
for(auto &pkg : packages) {
|
||||
pull_futures.emplace_back(std::async(std::launch::async, pull_package, std::ref(pkg), releases));
|
||||
}
|
||||
|
||||
for(auto &fut : pull_futures) {
|
||||
try {
|
||||
fut.get();
|
||||
log_info("Package pulled successfully.");
|
||||
} catch(std::exception &e) {
|
||||
log_error(std::string("Pull task generated an exception: ") + e.what());
|
||||
// Record the failure if not already recorded
|
||||
// (Handled inside pull_package)
|
||||
}
|
||||
}
|
||||
log_info("Completed Stage 1: All packages pulled.");
|
||||
|
||||
// Check for failures after Stage 1
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(failures_mutex);
|
||||
if(!failed_packages.empty()) {
|
||||
log_error("Failures detected after Stage 1: Pulling packages.");
|
||||
// Proceed to summary
|
||||
goto SUMMARY;
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 2: Build all packages in parallel
|
||||
log_info("Starting Stage 2: Building all packages.");
|
||||
std::vector<std::future<void>> build_futures;
|
||||
for(auto &pkg : packages) {
|
||||
build_futures.emplace_back(std::async(std::launch::async, build_package_stage_wrapper, std::ref(pkg), releases));
|
||||
}
|
||||
|
||||
for(auto &fut : build_futures) {
|
||||
try {
|
||||
fut.get();
|
||||
log_info("Package built successfully.");
|
||||
} catch(std::exception &e) {
|
||||
log_error(std::string("Build task generated an exception: ") + e.what());
|
||||
// Record the failure if not already recorded
|
||||
// (Handled inside build_package_stage_wrapper)
|
||||
}
|
||||
}
|
||||
log_info("Completed Stage 2: All packages built.");
|
||||
|
||||
// Check for failures after Stage 2
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(failures_mutex);
|
||||
if(!failed_packages.empty()) {
|
||||
log_error("Failures detected after Stage 2: Building packages.");
|
||||
// Proceed to summary
|
||||
goto SUMMARY;
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 3: Dput upload all packages in parallel
|
||||
log_info("Starting Stage 3: Uploading all packages with dput.");
|
||||
std::vector<std::future<void>> upload_futures;
|
||||
for(auto &pkg : packages) {
|
||||
upload_futures.emplace_back(std::async(std::launch::async, upload_package_stage, std::ref(pkg), skip_dput));
|
||||
}
|
||||
|
||||
for(auto &fut : upload_futures) {
|
||||
try {
|
||||
fut.get();
|
||||
log_info("Package uploaded successfully.");
|
||||
} catch(std::exception &e) {
|
||||
log_error(std::string("Upload task generated an exception: ") + e.what());
|
||||
// Record the failure if not already recorded
|
||||
// (Handled inside upload_package_stage)
|
||||
}
|
||||
}
|
||||
log_info("Completed Stage 3: All packages uploaded.");
|
||||
|
||||
// Check for failures after Stage 3
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(failures_mutex);
|
||||
if(!failed_packages.empty()) {
|
||||
log_error("Failures detected after Stage 3: Uploading packages.");
|
||||
// Proceed to summary
|
||||
goto SUMMARY;
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 4: Run Lintian on all packages in parallel
|
||||
log_info("Starting Stage 4: Running Lintian on all packages.");
|
||||
std::vector<std::future<void>> lintian_futures;
|
||||
for(auto &pkg : packages) {
|
||||
lintian_futures.emplace_back(std::async(std::launch::async, run_lintian_stage, std::ref(pkg)));
|
||||
}
|
||||
|
||||
for(auto &fut : lintian_futures) {
|
||||
try {
|
||||
fut.get();
|
||||
log_info("Lintian run successfully.");
|
||||
} catch(std::exception &e) {
|
||||
log_error(std::string("Lintian task generated an exception: ") + e.what());
|
||||
// Record the failure
|
||||
std::lock_guard<std::mutex> lock_fail(failures_mutex);
|
||||
failed_packages["Lintian"] = "Exception during Lintian run: " + std::string(e.what());
|
||||
}
|
||||
}
|
||||
log_info("Completed Stage 4: All Lintian runs completed.");
|
||||
|
||||
SUMMARY:
|
||||
if(!skip_cleanup) {
|
||||
log_info("Cleaning up output directory: " + OUTPUT_DIR);
|
||||
try {
|
||||
clean_old_logs(LOG_DIR); // Using common::clean_old_logs
|
||||
fs::remove_all(OUTPUT_DIR);
|
||||
log_info("Cleanup completed.");
|
||||
} catch(const std::exception &e) {
|
||||
log_error("Failed to clean up: " + std::string(e.what()));
|
||||
}
|
||||
} else {
|
||||
log_info("Skipping cleanup as per flag.");
|
||||
}
|
||||
|
||||
log_info("Publishing Lintian results.");
|
||||
publish_lintian();
|
||||
|
||||
log_info("Cleaning old logs.");
|
||||
try {
|
||||
clean_old_logs(LOG_DIR); // Using common::clean_old_logs
|
||||
} catch(const std::exception &e) {
|
||||
log_error("Failed to clean old logs: " + std::string(e.what()));
|
||||
}
|
||||
|
||||
// Summary of failures
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(failures_mutex);
|
||||
if(!failed_packages.empty()) {
|
||||
log_error("Summary of Failures:");
|
||||
for(const auto &entry : failed_packages) {
|
||||
log_error("Package: " + entry.first + " - Reason: " + entry.second);
|
||||
}
|
||||
std::cerr << "Some packages failed during processing. Check the log file for details.\n";
|
||||
return 1;
|
||||
} else {
|
||||
log_info("All packages processed successfully.");
|
||||
}
|
||||
}
|
||||
|
||||
log_info("Script completed successfully.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void log_all(const std::string &level, const std::string &msg, bool is_error=false) {
|
||||
// Function Definitions
|
||||
|
||||
static std::optional<std::string> run_lintian(const fs::path& source_path) {
|
||||
// Implement the run_lintian functionality or use existing utility functions
|
||||
// For the purpose of this example, we'll assume it's implemented elsewhere
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
static void log_all(const std::string &level, const std::string &msg, bool is_error) {
|
||||
std::string timestamp = get_current_utc_time();
|
||||
std::string full_msg = "[" + timestamp + "] [" + level + "] " + msg + "\n";
|
||||
|
||||
@ -152,7 +451,7 @@ static void print_help(const std::string &prog_name) {
|
||||
<< " --help, -h Display this help message.\n";
|
||||
}
|
||||
|
||||
static void run_command_silent_on_success(const std::vector<std::string> &cmd, const std::optional<fs::path> &cwd = std::nullopt) {
|
||||
static void run_command_silent_on_success(const std::vector<std::string> &cmd, const std::optional<fs::path> &cwd) {
|
||||
std::string command_str = std::accumulate(cmd.begin(), cmd.end(), std::string(),
|
||||
[](const std::string &a, const std::string &b) -> std::string { return a + (a.empty() ? "" : " ") + b; });
|
||||
|
||||
@ -164,19 +463,22 @@ static void run_command_silent_on_success(const std::vector<std::string> &cmd, c
|
||||
std::string exec_cmd = command_str;
|
||||
if(cwd) exec_cmd = "cd " + cwd->string() + " && " + exec_cmd;
|
||||
|
||||
FILE* pipe = popen(exec_cmd.c_str(), "r");
|
||||
// To handle the warning about ignoring attributes, use a lambda deleter with the correct signature
|
||||
auto deleter = [](FILE* ptr) { pclose(ptr); };
|
||||
std::unique_ptr<FILE, decltype(deleter)> pipe(popen(exec_cmd.c_str(), "r"), deleter);
|
||||
|
||||
if(!pipe) {
|
||||
log_error("Failed to run command: " + command_str);
|
||||
throw std::runtime_error("Command failed to start");
|
||||
}
|
||||
|
||||
std::stringstream ss;
|
||||
{
|
||||
char buffer[256];
|
||||
while(fgets(buffer, sizeof(buffer), pipe)) {
|
||||
ss << buffer;
|
||||
}
|
||||
char buffer[256];
|
||||
while(fgets(buffer, sizeof(buffer), pipe.get())) {
|
||||
ss << buffer;
|
||||
}
|
||||
int ret = pclose(pipe);
|
||||
|
||||
int ret = pclose(pipe.release());
|
||||
if (ret != 0) {
|
||||
log_error("Command failed with code " + std::to_string(ret) + ": " + command_str);
|
||||
log_error("Output:\n" + ss.str());
|
||||
@ -391,35 +693,43 @@ static void run_source_lintian(const std::string &name, const fs::path &source_p
|
||||
}
|
||||
log_verbose("Created Lintian suppression file: " + temp_file.string());
|
||||
std::string cmd = "lintian -EvIL +pedantic --suppress-tags-from-file " + temp_file.string() + " " + source_path.string() + " 2>&1";
|
||||
FILE* pipe = popen(cmd.c_str(), "r");
|
||||
|
||||
// Using the unique_ptr with correct deleter to avoid warnings
|
||||
auto deleter = [](FILE* ptr) { pclose(ptr); };
|
||||
std::unique_ptr<FILE, decltype(deleter)> pipe(popen(cmd.c_str(), "r"), deleter);
|
||||
|
||||
if(!pipe) {
|
||||
log_error("Failed to run Lintian command: " + cmd);
|
||||
// Record the failure
|
||||
std::lock_guard<std::mutex> lock_fail(failures_mutex);
|
||||
failed_packages[name] = "Failed to run Lintian command.";
|
||||
return;
|
||||
}
|
||||
|
||||
std::stringstream ss;
|
||||
if(pipe) {
|
||||
char buffer[256];
|
||||
while(fgets(buffer, sizeof(buffer), pipe)) {
|
||||
ss << buffer;
|
||||
}
|
||||
int ret = pclose(pipe);
|
||||
fs::remove(temp_file);
|
||||
log_verbose("Lintian command exited with code: " + std::to_string(ret));
|
||||
if(ret != 0) {
|
||||
log_error("Lintian reported issues for " + name + ":\n" + ss.str());
|
||||
if(!ss.str().empty()) {
|
||||
fs::path pkgdir = fs::path(BASE_LINTIAN_DIR) / name;
|
||||
fs::create_directories(pkgdir);
|
||||
std::ofstream out(pkgdir / "source.txt", std::ios::app);
|
||||
out << ss.str() << "\n";
|
||||
}
|
||||
} else {
|
||||
if(!ss.str().empty()) {
|
||||
fs::path pkgdir = fs::path(BASE_LINTIAN_DIR) / name;
|
||||
fs::create_directories(pkgdir);
|
||||
std::ofstream out(pkgdir / "source.txt", std::ios::app);
|
||||
out << ss.str() << "\n";
|
||||
}
|
||||
char buffer[256];
|
||||
while(fgets(buffer, sizeof(buffer), pipe.get())) {
|
||||
ss << buffer;
|
||||
}
|
||||
|
||||
int ret = pclose(pipe.release());
|
||||
fs::remove(temp_file);
|
||||
log_verbose("Lintian command exited with code: " + std::to_string(ret));
|
||||
if(ret != 0) {
|
||||
log_error("Lintian reported issues for " + name + ":\n" + ss.str());
|
||||
if(!ss.str().empty()) {
|
||||
fs::path pkgdir = fs::path(BASE_LINTIAN_DIR) / name;
|
||||
fs::create_directories(pkgdir);
|
||||
std::ofstream out(pkgdir / "source.txt", std::ios::app);
|
||||
out << ss.str() << "\n";
|
||||
}
|
||||
} else {
|
||||
fs::remove(temp_file);
|
||||
log_error("Failed to run Lintian for package: " + name);
|
||||
if(!ss.str().empty()) {
|
||||
fs::path pkgdir = fs::path(BASE_LINTIAN_DIR) / name;
|
||||
fs::create_directories(pkgdir);
|
||||
std::ofstream out(pkgdir / "source.txt", std::ios::app);
|
||||
out << ss.str() << "\n";
|
||||
}
|
||||
}
|
||||
log_verbose("Completed Lintian run for package: " + name);
|
||||
}
|
||||
@ -455,7 +765,7 @@ static void update_changelog(const fs::path &packaging_dir, const std::string &r
|
||||
} catch (const std::exception &e) {
|
||||
log_error("Failed to checkout debian/changelog for " + name + ": " + e.what());
|
||||
// Record the failure
|
||||
std::lock_guard<std::mutex> lock(failures_mutex);
|
||||
std::lock_guard<std::mutex> lock_fail(failures_mutex);
|
||||
failed_packages[name] = "Failed to checkout debian/changelog: " + std::string(e.what());
|
||||
throw;
|
||||
}
|
||||
@ -469,7 +779,7 @@ static void update_changelog(const fs::path &packaging_dir, const std::string &r
|
||||
} catch (const std::exception &e) {
|
||||
log_error("Failed to update changelog for " + name + ": " + e.what());
|
||||
// Record the failure
|
||||
std::lock_guard<std::mutex> lock(failures_mutex);
|
||||
std::lock_guard<std::mutex> lock_fail(failures_mutex);
|
||||
failed_packages[name] = "Failed to update changelog: " + std::string(e.what());
|
||||
throw;
|
||||
}
|
||||
@ -730,291 +1040,3 @@ static void run_lintian_stage(Package &pkg) {
|
||||
run_source_lintian(pkg.name, source_package_path);
|
||||
}
|
||||
}
|
||||
|
||||
static void clean_old_logs(const fs::path &log_dir) {
|
||||
// Implement log cleaning logic if necessary
|
||||
// Placeholder implementation
|
||||
log_info("Cleaning old logs in: " + log_dir.string());
|
||||
// Example: Remove logs older than 30 days
|
||||
auto now = std::chrono::system_clock::now();
|
||||
for(auto &p : fs::directory_iterator(log_dir)) {
|
||||
if(fs::is_regular_file(p)) {
|
||||
auto ftime = fs::last_write_time(p);
|
||||
auto sctp = std::chrono::time_point_cast<std::chrono::system_clock::duration>(ftime - fs::file_time_type::clock::now()
|
||||
+ std::chrono::system_clock::now());
|
||||
if(now - sctp > std::chrono::hours(24 * 30)) { // Older than 30 days
|
||||
fs::remove(p.path());
|
||||
log_verbose("Removed old log file: " + p.path().string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
std::string prog_name = fs::path(argv[0]).filename().string();
|
||||
for(int i = 1; i < argc; i++) {
|
||||
std::string arg = argv[i];
|
||||
if(arg == "--help" || arg == "-h") {
|
||||
print_help(prog_name);
|
||||
return 0;
|
||||
}
|
||||
if(arg == "--verbose" || arg == "-v") {
|
||||
verbose = true;
|
||||
// Remove the verbose flag from argv
|
||||
for(int j = i; j < argc - 1; j++) {
|
||||
argv[j] = argv[j+1];
|
||||
}
|
||||
argc--;
|
||||
i--;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
log_info("Script started.");
|
||||
fs::create_directories(LOG_DIR);
|
||||
log_info("Ensured log directory exists: " + LOG_DIR);
|
||||
fs::create_directories(OUTPUT_DIR);
|
||||
log_info("Ensured output directory exists: " + OUTPUT_DIR);
|
||||
|
||||
auto now = std::time(nullptr);
|
||||
std::tm tm;
|
||||
gmtime_r(&now, &tm);
|
||||
char buf_time[20];
|
||||
std::strftime(buf_time, sizeof(buf_time), "%Y%m%dT%H%M%S", &tm);
|
||||
std::string current_time = buf_time;
|
||||
|
||||
std::string uuid_part = current_time.substr(0,10);
|
||||
BASE_LINTIAN_DIR = BASE_OUTPUT_DIR + "/.lintian.tmp." + uuid_part;
|
||||
fs::create_directories(BASE_LINTIAN_DIR);
|
||||
log_info("Created Lintian temporary directory: " + BASE_LINTIAN_DIR);
|
||||
|
||||
fs::path log_file = fs::path(LOG_DIR) / (current_time + ".log");
|
||||
log_info("Opening log file: " + log_file.string());
|
||||
log_file_stream.open(log_file);
|
||||
if(!log_file_stream.is_open()) {
|
||||
std::cerr << "[ERROR] Unable to open log file.\n";
|
||||
return 1;
|
||||
}
|
||||
log_info("Log file opened successfully.");
|
||||
|
||||
bool skip_dput = false;
|
||||
bool skip_cleanup = false;
|
||||
std::string config_path;
|
||||
for(int i = 1; i < argc; i++) {
|
||||
std::string arg = argv[i];
|
||||
log_info("Processing argument: " + arg);
|
||||
if(arg == "--skip-dput") {
|
||||
skip_dput = true;
|
||||
log_info("Flag set: --skip-dput");
|
||||
} else if(arg == "--skip-cleanup") {
|
||||
skip_cleanup = true;
|
||||
log_info("Flag set: --skip-cleanup");
|
||||
} else if(arg.rfind("--urgency-level=", 0) == 0) {
|
||||
urgency_level_override = arg.substr(std::string("--urgency-level=").size());
|
||||
log_info("Urgency level overridden to: " + urgency_level_override);
|
||||
} else if(arg.rfind("--workers=", 0) == 0) {
|
||||
worker_count = std::stoi(arg.substr(std::string("--workers=").size()));
|
||||
if(worker_count < 1) worker_count = 1;
|
||||
log_info("Worker count set to: " + std::to_string(worker_count));
|
||||
} else if(config_path.empty()) {
|
||||
config_path = arg;
|
||||
log_info("Config path set to: " + config_path);
|
||||
}
|
||||
}
|
||||
|
||||
if(config_path.empty()) {
|
||||
log_error("No config file specified.");
|
||||
print_help(prog_name);
|
||||
return 1;
|
||||
}
|
||||
|
||||
setenv("DEBFULLNAME", DEBFULLNAME.c_str(), 1);
|
||||
log_info("Set DEBFULLNAME to: " + DEBFULLNAME);
|
||||
setenv("DEBEMAIL", DEBEMAIL.c_str(), 1);
|
||||
log_info("Set DEBEMAIL to: " + DEBEMAIL);
|
||||
|
||||
YAML::Node config;
|
||||
try {
|
||||
config = load_config(config_path);
|
||||
} catch (std::exception &e) {
|
||||
log_error(std::string("Error loading config file: ") + e.what());
|
||||
return 1;
|
||||
}
|
||||
|
||||
auto packages_node = config["packages"];
|
||||
auto releases = config["releases"];
|
||||
log_info("Loaded " + std::to_string(packages_node.size()) + " packages and " + std::to_string(releases.size()) + " releases from config.");
|
||||
|
||||
// Populate the packages vector
|
||||
std::vector<Package> packages;
|
||||
for(auto pkg_node : packages_node) {
|
||||
Package pkg;
|
||||
pkg.name = pkg_node["name"] ? pkg_node["name"].as<std::string>() : "";
|
||||
pkg.upload_target = pkg_node["upload_target"] ? pkg_node["upload_target"].as<std::string>() : "ppa:lubuntu-ci/unstable-ci-proposed";
|
||||
pkg.upstream_url = pkg_node["upstream_url"] ? pkg_node["upstream_url"].as<std::string>() : ("https://github.com/lxqt/" + pkg.name + ".git");
|
||||
pkg.packaging_url = pkg_node["packaging_url"] ? pkg_node["packaging_url"].as<std::string>() : ("https://git.lubuntu.me/Lubuntu/" + pkg.name + "-packaging.git");
|
||||
if(pkg_node["packaging_branch"] && pkg_node["packaging_branch"].IsScalar()) {
|
||||
pkg.packaging_branch = pkg_node["packaging_branch"].as<std::string>();
|
||||
}
|
||||
pkg.large = pkg_node["large"] ? pkg_node["large"].as<bool>() : false;
|
||||
|
||||
if(pkg.name.empty()) {
|
||||
log_warning("Skipping package due to missing name.");
|
||||
continue;
|
||||
}
|
||||
packages.emplace_back(std::move(pkg));
|
||||
}
|
||||
log_info("Prepared " + std::to_string(packages.size()) + " packages for processing.");
|
||||
|
||||
fs::current_path(BASE_DIR);
|
||||
log_info("Set current working directory to BASE_DIR: " + BASE_DIR);
|
||||
|
||||
// Stage 1: Pull all packages in parallel
|
||||
log_info("Starting Stage 1: Pulling all packages.");
|
||||
std::vector<std::future<void>> pull_futures;
|
||||
for(auto &pkg : packages) {
|
||||
pull_futures.emplace_back(std::async(std::launch::async, pull_package, std::ref(pkg), releases));
|
||||
}
|
||||
|
||||
for(auto &fut : pull_futures) {
|
||||
try {
|
||||
fut.get();
|
||||
log_info("Package pulled successfully.");
|
||||
} catch(std::exception &e) {
|
||||
log_error(std::string("Pull task generated an exception: ") + e.what());
|
||||
// Record the failure if not already recorded
|
||||
// (Handled inside pull_package)
|
||||
}
|
||||
}
|
||||
log_info("Completed Stage 1: All packages pulled.");
|
||||
|
||||
// Check for failures after Stage 1
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(failures_mutex);
|
||||
if(!failed_packages.empty()) {
|
||||
log_error("Failures detected after Stage 1: Pulling packages.");
|
||||
// Skip remaining stages
|
||||
goto SUMMARY;
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 2: Build all packages in parallel
|
||||
log_info("Starting Stage 2: Building all packages.");
|
||||
std::vector<std::future<void>> build_futures;
|
||||
for(auto &pkg : packages) {
|
||||
build_futures.emplace_back(std::async(std::launch::async, build_package_stage_wrapper, std::ref(pkg), releases));
|
||||
}
|
||||
|
||||
for(auto &fut : build_futures) {
|
||||
try {
|
||||
fut.get();
|
||||
log_info("Package built successfully.");
|
||||
} catch(std::exception &e) {
|
||||
log_error(std::string("Build task generated an exception: ") + e.what());
|
||||
// Record the failure if not already recorded
|
||||
// (Handled inside build_package_stage_wrapper)
|
||||
}
|
||||
}
|
||||
log_info("Completed Stage 2: All packages built.");
|
||||
|
||||
// Check for failures after Stage 2
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(failures_mutex);
|
||||
if(!failed_packages.empty()) {
|
||||
log_error("Failures detected after Stage 2: Building packages.");
|
||||
// Skip remaining stages
|
||||
goto SUMMARY;
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 3: Dput upload all packages in parallel
|
||||
log_info("Starting Stage 3: Uploading all packages with dput.");
|
||||
std::vector<std::future<void>> upload_futures;
|
||||
for(auto &pkg : packages) {
|
||||
upload_futures.emplace_back(std::async(std::launch::async, upload_package_stage, std::ref(pkg), skip_dput));
|
||||
}
|
||||
|
||||
for(auto &fut : upload_futures) {
|
||||
try {
|
||||
fut.get();
|
||||
log_info("Package uploaded successfully.");
|
||||
} catch(std::exception &e) {
|
||||
log_error(std::string("Upload task generated an exception: ") + e.what());
|
||||
// Record the failure if not already recorded
|
||||
// (Handled inside upload_package_stage)
|
||||
}
|
||||
}
|
||||
log_info("Completed Stage 3: All packages uploaded.");
|
||||
|
||||
// Check for failures after Stage 3
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(failures_mutex);
|
||||
if(!failed_packages.empty()) {
|
||||
log_error("Failures detected after Stage 3: Uploading packages.");
|
||||
// Skip remaining stages
|
||||
goto SUMMARY;
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 4: Run Lintian on all packages in parallel
|
||||
log_info("Starting Stage 4: Running Lintian on all packages.");
|
||||
std::vector<std::future<void>> lintian_futures;
|
||||
for(auto &pkg : packages) {
|
||||
lintian_futures.emplace_back(std::async(std::launch::async, run_lintian_stage, std::ref(pkg)));
|
||||
}
|
||||
|
||||
for(auto &fut : lintian_futures) {
|
||||
try {
|
||||
fut.get();
|
||||
log_info("Lintian run successfully.");
|
||||
} catch(std::exception &e) {
|
||||
log_error(std::string("Lintian task generated an exception: ") + e.what());
|
||||
// Record the failure
|
||||
std::lock_guard<std::mutex> lock_fail(failures_mutex);
|
||||
failed_packages["Lintian"] = "Exception during Lintian run: " + std::string(e.what());
|
||||
}
|
||||
}
|
||||
log_info("Completed Stage 4: All Lintian runs completed.");
|
||||
|
||||
// Check for failures after Stage 4
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(failures_mutex);
|
||||
if(!failed_packages.empty()) {
|
||||
log_error("Failures detected after Stage 4: Running Lintian.");
|
||||
// Proceed to summary
|
||||
}
|
||||
}
|
||||
|
||||
SUMMARY:
|
||||
if(!skip_cleanup) {
|
||||
log_info("Cleaning up output directory: " + OUTPUT_DIR);
|
||||
fs::remove_all(OUTPUT_DIR);
|
||||
log_info("Cleanup completed.");
|
||||
} else {
|
||||
log_info("Skipping cleanup as per flag.");
|
||||
}
|
||||
|
||||
log_info("Publishing Lintian results.");
|
||||
publish_lintian();
|
||||
|
||||
log_info("Cleaning old logs.");
|
||||
clean_old_logs(fs::path(LOG_DIR));
|
||||
|
||||
// Summary of failures
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(failures_mutex);
|
||||
if(!failed_packages.empty()) {
|
||||
log_error("Summary of Failures:");
|
||||
for(const auto &entry : failed_packages) {
|
||||
log_error("Package: " + entry.first + " - Reason: " + entry.second);
|
||||
}
|
||||
std::cerr << "Some packages failed during processing. Check the log file for details.\n";
|
||||
return 1;
|
||||
} else {
|
||||
log_info("All packages processed successfully.");
|
||||
}
|
||||
}
|
||||
|
||||
log_info("Script completed successfully.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user