From 06a6567da993135cabe814876977f5f32a0cb25c Mon Sep 17 00:00:00 2001 From: Simon Quigley Date: Sat, 1 Feb 2025 17:02:05 -0600 Subject: [PATCH] Various cleanup bits --- cpp/CMakeLists.txt | 2 - cpp/ci_database_objs.h | 2 +- cpp/ci_logic.cpp | 39 ++++- cpp/ci_logic.h | 157 ++++++++++-------- cpp/common.cpp | 339 -------------------------------------- cpp/common.h | 89 ---------- cpp/lintian-ppa.cpp | 2 +- cpp/lubuntuci_lib.cpp | 82 --------- cpp/lubuntuci_lib.h | 48 ------ cpp/task_queue.cpp | 121 +++++--------- cpp/update-maintainer.cpp | 6 +- cpp/utilities.cpp | 315 ++++++++++++++++++++++++++++++++++- cpp/utilities.h | 74 ++++++++- cpp/web_server.cpp | 121 ++++++-------- 14 files changed, 595 insertions(+), 802 deletions(-) delete mode 100644 cpp/common.cpp delete mode 100644 cpp/common.h delete mode 100644 cpp/lubuntuci_lib.cpp delete mode 100644 cpp/lubuntuci_lib.h diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 9f9d75f..b7d91ed 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -40,14 +40,12 @@ set(UUID_LIB "uuid") # 1. The main library: lubuntuci_lib # add_library(lubuntuci_lib SHARED - common.cpp utilities.cpp db_common.cpp git_common.cpp sources_parser.cpp ci_logic.cpp ci_database_objs.cpp - lubuntuci_lib.cpp task_queue.cpp template_renderer.cpp web_server.cpp diff --git a/cpp/ci_database_objs.h b/cpp/ci_database_objs.h index c6fabcd..3913a08 100644 --- a/cpp/ci_database_objs.h +++ b/cpp/ci_database_objs.h @@ -27,7 +27,7 @@ #include #include -#include "common.h" +#include "utilities.h" class Person { public: diff --git a/cpp/ci_logic.cpp b/cpp/ci_logic.cpp index d6bafb8..5bbf56d 100644 --- a/cpp/ci_logic.cpp +++ b/cpp/ci_logic.cpp @@ -6,7 +6,7 @@ // (at your option) any later version. // // This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of +// but WITHOUT ANY WARRANTY, without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // @@ -15,8 +15,6 @@ #include "task_queue.h" #include "ci_logic.h" -#include "lubuntuci_lib.h" -#include "common.h" #include "utilities.h" #include "db_common.h" #include "git_common.h" @@ -530,7 +528,7 @@ void CiLogic::process_entire_pipeline(std::shared_ptr &proj, bool pull_success = pull_project(proj); bool tarball_success = create_project_tarball(proj); const auto [build_success, changes_files] = build_project(proj); - upload_and_lint(proj, changes_files, skip_dput); + upload_and_lint(proj, changes_files, skip_dput, std::make_shared()); do_summary(skip_cleanup); log_info("Pipeline done for " + proj->package->name); } catch(std::exception &ex) { @@ -829,3 +827,36 @@ std::string CiLogic::get_task_log(int task_id) { } return ""; } + +std::vector> CiLogic::list_known_repos(int page, + int per_page, + const std::string &sort_by, + const std::string &sort_order) { + init_global(); + if (page != 0 && per_page != 0 && !sort_by.empty() && !sort_order.empty()) return get_config("", page, per_page, sort_by, sort_order); + else return get_config(); +} + +bool CiLogic::pull_repo_by_name(const std::string &repo_name, std::shared_ptr log) { + init_global(); + auto pkgconfs = get_config(repo_name); + if (pkgconfs.empty()) return false; + return pull_project(pkgconfs.at(0), log); +} + +bool CiLogic::create_project_tarball_by_name(const std::string &repo_name, std::shared_ptr log) { + init_global(); + auto pkgconfs = get_config(repo_name); + if (pkgconfs.empty()) return false; + return create_project_tarball(pkgconfs.at(0), log); +} + +bool CiLogic::build_repo_by_name(const std::string &repo_name, std::shared_ptr log) { + init_global(); + bool success = true; + for (auto pkgconf : get_config(repo_name)) { + auto [build_ok, changes_files] = build_project(pkgconf, log); + success = success && build_ok && upload_and_lint(pkgconf, changes_files, false, log); + } + return success; +} diff --git a/cpp/ci_logic.h b/cpp/ci_logic.h index 6e52e64..408321e 100644 --- a/cpp/ci_logic.h +++ b/cpp/ci_logic.h @@ -32,11 +32,8 @@ #include #include -struct CiProject; +namespace fs = std::filesystem; -/** - * Data describing one package to pull/build/etc. - */ struct CiProject { std::string name; std::string version; @@ -45,78 +42,98 @@ struct CiProject { std::string upstream_url; std::string packaging_url; std::optional packaging_branch; - std::filesystem::path main_tarball; + fs::path main_tarball; bool large = false; - - // These get populated during build + // These get populated during build: std::vector changes_files; std::vector devel_changes_files; }; class CiLogic { - public: - // Initialize global configurations - void init_global(); - - // Load YAML configuration from a given path - YAML::Node load_yaml_config(const std::filesystem::path &config_path); - - // Convert a YAML node to a CiProject structure - CiProject yaml_to_project(const YAML::Node &pkg_node); - - bool pull_project(std::shared_ptr &proj, std::shared_ptr log = NULL); - bool create_project_tarball(std::shared_ptr &proj, std::shared_ptr log = NULL); - std::tuple> build_project(std::shared_ptr proj, std::shared_ptr log = NULL); - bool upload_and_lint(std::shared_ptr &proj, const std::set changes_files, bool skip_dput, std::shared_ptr log = NULL); - - // Perform cleanup and summarize the build process - void do_summary(bool skip_cleanup); - - // Process the entire pipeline for a given PackageConf ID - void process_entire_pipeline(std::shared_ptr &proj, bool skip_dput, bool skip_cleanup); - - // Retrieve all PackageConf entries from the database - std::vector> get_config(const std::string &repo_name = "", int page = 0, int per_page = 0, const std::string& sort_by = "", const std::string& sort_order = ""); - - // Function to enqueue tasks - void enqueue(std::function task); - - std::shared_ptr>> get_job_statuses(); - std::vector> get_packageconfs(); - std::shared_ptr get_packageconf_by_id(int id); - std::vector> get_packageconfs_by_ids(std::set ids); - void set_packageconfs(std::vector> _pkgconfs); - void sync(std::shared_ptr pkgconf); - - std::string queue_pull_tarball(std::vector> repos, - std::unique_ptr& task_queue, - std::shared_ptr>> job_statuses); - std::string queue_build_upload(std::vector> repos, - std::unique_ptr& task_queue, - std::shared_ptr>> job_statuses); - - std::string get_task_log(int task_id); - - std::vector releases; - std::vector packages; - std::vector branches; - - private: - void debuild_package(const fs::path &packaging_dir, std::shared_ptr log); - - QSqlDatabase p_db; - - mutable std::mutex packageconfs_mutex_; - std::vector> packageconfs; - std::shared_ptr>> _cached_job_statuses; - - struct package_conf_item { - std::shared_ptr first_pkgconf; - std::shared_ptr first_pull_task = std::make_shared(); - std::shared_ptr first_tarball_task = std::make_shared(); - std::shared_ptr packaging_commit = std::make_shared(); - std::shared_ptr upstream_commit = std::make_shared(); - }; +public: + // Initialize global config and database + void init_global(); + + // Load YAML config from a given path + YAML::Node load_yaml_config(const fs::path &config_path); + + // Convert a YAML node to a CiProject + CiProject yaml_to_project(const YAML::Node &pkg_node); + + // Pipeline functions + bool pull_project(std::shared_ptr &proj, std::shared_ptr log = nullptr); + bool create_project_tarball(std::shared_ptr &proj, std::shared_ptr log = nullptr); + std::tuple> build_project(std::shared_ptr proj, std::shared_ptr log = nullptr); + bool upload_and_lint(std::shared_ptr &proj, + const std::set changes_files, + bool skip_dput, + std::shared_ptr log = nullptr); + + // Summary & cleanup + void do_summary(bool skip_cleanup); + + // Orchestrate entire pipeline + void process_entire_pipeline(std::shared_ptr &proj, + bool skip_dput, + bool skip_cleanup); + + // Retrieve PackageConf entries (with optional pagination/sorting) + std::vector> get_config(const std::string &repo_name = "", + int page = 0, + int per_page = 0, + const std::string &sort_by = "", + const std::string &sort_order = ""); + + // Enqueue a task (wrapper) + void enqueue(std::function task); + + // Job status and PackageConf getters + std::shared_ptr>> get_job_statuses(); + std::vector> get_packageconfs(); + std::shared_ptr get_packageconf_by_id(int id); + std::vector> get_packageconfs_by_ids(std::set ids); + void set_packageconfs(std::vector> _pkgconfs); + void sync(std::shared_ptr pkgconf); + + // Queue tasks + std::string queue_pull_tarball(std::vector> repos, + std::unique_ptr& task_queue, + std::shared_ptr>> job_statuses); + std::string queue_build_upload(std::vector> repos, + std::unique_ptr& task_queue, + std::shared_ptr>> job_statuses); + + // Get a task’s log + std::string get_task_log(int task_id); + + std::vector> list_known_repos(int page = 0, + int per_page = 0, + const std::string &sort_by = "", + const std::string &sort_order = ""); + bool pull_repo_by_name(const std::string &repo_name, std::shared_ptr log = nullptr); + bool create_project_tarball_by_name(const std::string &repo_name, std::shared_ptr log = nullptr); + bool build_repo_by_name(const std::string &repo_name, std::shared_ptr log = nullptr); + + // These come from the config/DB + std::vector releases; + std::vector packages; + std::vector branches; + +private: + void debuild_package(const fs::path &packaging_dir, std::shared_ptr log); + + QSqlDatabase p_db; + mutable std::mutex packageconfs_mutex_; + std::vector> packageconfs; + std::shared_ptr>> _cached_job_statuses; + + struct package_conf_item { + std::shared_ptr first_pkgconf; + std::shared_ptr first_pull_task = std::make_shared(); + std::shared_ptr first_tarball_task = std::make_shared(); + std::shared_ptr packaging_commit = std::make_shared(); + std::shared_ptr upstream_commit = std::make_shared(); + }; }; #endif // CI_LOGIC_H diff --git a/cpp/common.cpp b/cpp/common.cpp deleted file mode 100644 index bdea803..0000000 --- a/cpp/common.cpp +++ /dev/null @@ -1,339 +0,0 @@ -// Copyright (C) 2024-2025 Simon Quigley -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -#include "common.h" -#include "utilities.h" -#include "/usr/include/archive.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// Define the global 'verbose' variable -bool verbose = false; - -// Logger function implementations -void log_info(const std::string &msg) { - std::cout << "[INFO] " << msg << "\n"; -} - -void log_warning(const std::string &msg) { - std::cerr << "[WARNING] " << msg << "\n"; -} - -void log_error(const std::string &msg) { - std::cerr << "[ERROR] " << msg << "\n"; -} - -void log_verbose(const std::string &msg) { - if (verbose) { - std::cout << "[VERBOSE] " << msg << "\n"; - } -} - -namespace fs = std::filesystem; - -bool run_command(const std::vector &cmd, - const std::optional &cwd, - bool show_output, - std::shared_ptr log) { - if (cmd.empty()) { - throw std::runtime_error("Command is empty"); - } - - QProcess process; - - // Set the working directory if provided - if (cwd) { - process.setWorkingDirectory(QString::fromStdString(cwd->string())); - } - - // Set up the environment (if needed) - QProcessEnvironment env = QProcessEnvironment::systemEnvironment(); - process.setProcessEnvironment(env); - - // Extract executable and arguments - QString program = QString::fromStdString(cmd[0]); - QStringList arguments; - for (size_t i = 1; i < cmd.size(); ++i) { - arguments << QString::fromStdString(cmd[i]); - } - - // Start the command - process.start(program, arguments); - if (!process.waitForStarted()) { - throw std::runtime_error("Failed to start the command: " + program.toStdString()); - } - - // Stream output while the process is running - while (process.state() == QProcess::Running) { - if (process.waitForReadyRead()) { - QByteArray output = process.readAllStandardOutput(); - QByteArray error = process.readAllStandardError(); - - if (log) { - log->append(output.toStdString()); - log->append(error.toStdString()); - } - - if (show_output) { - std::cout << output.toStdString(); - std::cerr << error.toStdString(); - } - } - } - - // Wait for the process to finish - process.waitForFinished(); - - // Capture return code and errors - if (process.exitStatus() != QProcess::NormalExit || process.exitCode() != 0) { - QByteArray error_output = process.readAllStandardError(); - std::string error_message = "Command failed with exit code: " + std::to_string(process.exitCode()); - if (!error_output.isEmpty()) { - error_message += "\nError Output: " + error_output.toStdString(); - } - throw std::runtime_error(error_message); - } - - return true; -} - -// Function to extract excluded files from a copyright file -std::vector extract_files_excluded(const std::string& filepath) { - std::ifstream file(filepath); - if (!file.is_open()) { - throw std::runtime_error("Failed to open file: " + filepath); - } - - std::vector files_excluded; - std::string line; - std::regex files_excluded_pattern(R"(Files-Excluded:\s*(.*))"); - bool in_files_excluded = false; - - while (std::getline(file, line)) { - if (std::regex_match(line, files_excluded_pattern)) { - in_files_excluded = true; - std::smatch match; - if (std::regex_search(line, match, files_excluded_pattern) && match.size() > 1) { - files_excluded.emplace_back(match[1]); - } - } else if (in_files_excluded) { - if (!line.empty() && (line[0] == ' ' || line[0] == '\t')) { - files_excluded.emplace_back(line.substr(1)); - } else { - break; // End of Files-Excluded block - } - } - } - - return files_excluded; -} - -// Function to create a tarball -void create_tarball(const std::string& tarballPath, const std::string& directory, const std::vector& exclusions, std::shared_ptr log) { - log->append("Creating tarball: " + tarballPath); - - struct archive* a = archive_write_new(); - if (!a) { - throw std::runtime_error("Failed to create a new archive."); - } - - if (archive_write_add_filter_gzip(a) != ARCHIVE_OK) { - std::string err = "Failed to add gzip filter: "; - err += archive_error_string(a); - archive_write_free(a); - throw std::runtime_error(err); - } - - if (archive_write_set_format_pax_restricted(a) != ARCHIVE_OK) { - std::string err = "Failed to set format: "; - err += archive_error_string(a); - archive_write_free(a); - throw std::runtime_error(err); - } - - if (archive_write_open_filename(a, tarballPath.c_str()) != ARCHIVE_OK) { - std::string err = "Could not open tarball for writing: "; - err += archive_error_string(a); - archive_write_free(a); - throw std::runtime_error(err); - } - - // Initialize a set to track added relative paths to prevent duplication - std::unordered_set added_paths; - - // Iterate through the directory recursively without following symlinks - for (auto it = fs::recursive_directory_iterator( - directory, - fs::directory_options::skip_permission_denied); - it != fs::recursive_directory_iterator(); ++it) { - const auto& path = it->path(); - std::error_code ec; - - fs::path relative_path = fs::relative(path, directory, ec); - if (ec) { - log->append("Failed to compute relative path for: " + path.string() + " Error: " + ec.message()); - continue; - } - - // Normalize the relative path to avoid discrepancies - fs::path normalized_relative_path = relative_path.lexically_normal(); - std::string relative_path_str = normalized_relative_path.string(); - - // Check if this path has already been added - if (!added_paths.insert(relative_path_str).second) { - log->append("Duplicate path detected and skipped: " + relative_path_str); - continue; // Skip adding this duplicate path - } - - // Exclusion logic (if any exclusions are provided) - bool excluded = std::any_of(exclusions.begin(), exclusions.end(), [&relative_path_str](const std::string& exclusion) { - return relative_path_str.find(exclusion) != std::string::npos; - }); - if (excluded) { continue; } - - fs::file_status fstatus = it->symlink_status(ec); - if (ec) { - log->append("Failed to get file status for: " + path.string() + " Error: " + ec.message()); - continue; - } - - struct archive_entry* entry = archive_entry_new(); - if (!entry) { - log->append("Failed to create archive entry for: " + path.string()); - archive_write_free(a); - throw std::runtime_error("Failed to create archive entry."); - } - - std::string entry_path = relative_path_str; - if (fs::is_directory(fstatus)) { - // Ensure the directory pathname ends with '/' - if (!entry_path.empty() && entry_path.back() != '/') { - entry_path += '/'; - } - archive_entry_set_pathname(entry, entry_path.c_str()); - } else { - archive_entry_set_pathname(entry, entry_path.c_str()); - } - - // Set file type, permissions, and size - if (fs::is_regular_file(fstatus)) { - // Regular file - uintmax_t filesize = fs::file_size(path, ec); - if (ec) { - log->append("Cannot get file size for: " + path.string() + " Error: " + ec.message()); - archive_entry_free(entry); - continue; - } - archive_entry_set_size(entry, static_cast(filesize)); - archive_entry_set_filetype(entry, AE_IFREG); - archive_entry_set_perm(entry, static_cast(fstatus.permissions())); - } - else if (fs::is_symlink(fstatus)) { - fs::path target = fs::read_symlink(path, ec); - if (ec) { - log->append("Cannot read symlink for: " + path.string() + " Error: " + ec.message()); - archive_entry_free(entry); - continue; - } - archive_entry_set_symlink(entry, target.c_str()); - archive_entry_set_filetype(entry, AE_IFLNK); - archive_entry_set_perm(entry, static_cast(fstatus.permissions())); - } - else if (fs::is_directory(fstatus)) { - archive_entry_set_size(entry, 0); - archive_entry_set_filetype(entry, AE_IFDIR); - archive_entry_set_perm(entry, static_cast(fstatus.permissions())); - } - else { - log->append("Unsupported file type for: " + path.string()); - archive_entry_free(entry); - continue; - } - - // Retrieve and set the modification time - fs::file_time_type ftime = fs::last_write_time(path, ec); - std::time_t mtime; - if (ec) { - log->append("Failed to get last write time for: " + path.string() + " Error: " + ec.message()); - // Obtain current UTC time as fallback - auto now = std::chrono::system_clock::now(); - mtime = std::chrono::system_clock::to_time_t(now); - log->append("Setting default mtime (current UTC time) for: " + path.string()); - } else { - mtime = to_time_t(ftime); - } - archive_entry_set_mtime(entry, mtime, 0); - - if (archive_write_header(a, entry) != ARCHIVE_OK) { - log->append("Failed to write header for: " + path.string() + " Error: " + archive_error_string(a)); - archive_entry_free(entry); - continue; - } - - if (fs::is_regular_file(fstatus)) { - std::ifstream fileStream(path, std::ios::binary); - if (!fileStream) { - log->append("Failed to open file for reading: " + path.string()); - archive_entry_free(entry); - continue; - } - - const std::size_t bufferSize = 8192; - char buffer[bufferSize]; - while (fileStream) { - fileStream.read(buffer, bufferSize); - std::streamsize bytesRead = fileStream.gcount(); - if (bytesRead > 0) { - if (archive_write_data(a, buffer, static_cast(bytesRead)) < 0) { - log->append("Failed to write data for: " + path.string() + " Error: " + archive_error_string(a)); - break; - } - } - } - - if (fileStream.bad()) { - log->append("Error reading file: " + path.string()); - } - } - - archive_entry_free(entry); - } - - if (archive_write_close(a) != ARCHIVE_OK) { - std::string err = "Failed to close archive: "; - err += archive_error_string(a); - archive_write_free(a); - throw std::runtime_error(err); - } - - if (archive_write_free(a) != ARCHIVE_OK) { - std::string err = "Failed to free archive: "; - err += archive_error_string(a); - throw std::runtime_error(err); - } - - log->append("Tarball created and compressed: " + tarballPath); -} diff --git a/cpp/common.h b/cpp/common.h deleted file mode 100644 index 7dd1557..0000000 --- a/cpp/common.h +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (C) 2024 Simon Quigley -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -#ifndef COMMON_H -#define COMMON_H - -#include "utilities.h" -#include -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; -class Task; - -class Log { -private: - std::string data = ""; - mutable std::shared_mutex lock_; - std::weak_ptr task_context_; - std::string last_data_str = ""; - -public: - void append(const std::string& str) { - std::unique_lock lock(lock_); - std::string log_str = str.ends_with('\n') ? str : str + '\n'; - if (str.empty() || last_data_str == log_str) { return; } - else if (str.contains("dpkg-source: warning: ignoring deletion of file")) { return; } - data += std::format("[{}] {}", get_current_utc_time("%Y-%m-%dT%H:%M:%SZ"), log_str); - last_data_str = log_str; - } - - void set_log(const std::string& str) { - std::unique_lock lock(lock_); - data = str; - } - - std::string get() const { - std::shared_lock lock(lock_); - return std::regex_replace(data, std::regex(R"(^\s+)"), ""); - } - - void assign_task_context(std::shared_ptr task) { - task_context_ = task; - } - - std::shared_ptr get_task_context() const { - return task_context_.lock(); - } -}; - -// Logger functions -extern bool verbose; -void log_info(const std::string &msg); -void log_warning(const std::string &msg); -void log_error(const std::string &msg); -void log_verbose(const std::string &msg); - -// Function to run a command with optional working directory and show output -bool run_command(const std::vector &cmd, - const std::optional &cwd = std::nullopt, - bool show_output = false, - std::shared_ptr log = nullptr); - -// Function to extract excluded files from a copyright file -std::vector extract_files_excluded(const std::string& filepath); - -// Function to create a tarball -void create_tarball(const std::string& tarballPath, - const std::string& directory, - const std::vector& exclusions, - std::shared_ptr log = nullptr); - -#endif // COMMON_H diff --git a/cpp/lintian-ppa.cpp b/cpp/lintian-ppa.cpp index 6374b1e..15a93b5 100644 --- a/cpp/lintian-ppa.cpp +++ b/cpp/lintian-ppa.cpp @@ -1,4 +1,4 @@ -#include "common.h" +#include "utilities.h" #include "ci_logic.h" #include #include diff --git a/cpp/lubuntuci_lib.cpp b/cpp/lubuntuci_lib.cpp deleted file mode 100644 index 0e7f872..0000000 --- a/cpp/lubuntuci_lib.cpp +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (C) 2024-2025 Simon Quigley -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -#include "lubuntuci_lib.h" -#include "ci_logic.h" -#include "common.h" -#include -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; - -/** - * list_known_repos(): - * Make sure we call CiLogic::init_global() before reading - * the config, otherwise the config node will be empty. - */ -std::vector> LubuntuCI::list_known_repos(int page, int per_page, const std::string& sort_by, const std::string& sort_order) -{ - cilogic.init_global(); - if (page == 0 || per_page == 0 || sort_by.empty() || sort_order.empty()) { return cilogic.get_config(); } - return cilogic.get_config("", page, per_page, sort_by, sort_order); -} - -/** - * pull_repo(): - * - We do not call init_global() here because list_known_repos() - * or build_repo() might do it. But calling it again is safe. - */ -bool LubuntuCI::pull_repo(const std::string &repo_name, std::shared_ptr log) -{ - log->append("Ensuring the global config is initialized...\n"); - cilogic.init_global(); - log->append("Global config is initialized. Getting the configs for the package name...\n"); - auto pkgconfs = cilogic.get_config(repo_name); - log->append("Configs retrieved. Performing the pull...\n"); - return cilogic.pull_project(pkgconfs.at(0), log); -} - -/** - * create_project_tarball - */ -bool LubuntuCI::create_project_tarball(const std::string &repo_name, std::shared_ptr log) -{ - cilogic.init_global(); - log->append("Global config is initialized. Getting the configs for the package name...\n"); - auto pkgconfs = cilogic.get_config(repo_name); - log->append("Configs retrieved. Performing the tarball creation...\n"); - return cilogic.create_project_tarball(pkgconfs.at(0), log); -} - -/** - * build_repo(): - * - Also safely calls init_global(). - * - Reads skip_dput from config if present (default = false). - */ -bool LubuntuCI::build_repo(const std::string &repo_name, std::shared_ptr log) -{ - cilogic.init_global(); - bool success = true; - for (auto pkgconf : cilogic.get_config(repo_name)) { - const auto [build_success, changes_files] = cilogic.build_project(pkgconf, log); - success = success && build_success && cilogic.upload_and_lint(pkgconf, changes_files, false); - } - return success; -} diff --git a/cpp/lubuntuci_lib.h b/cpp/lubuntuci_lib.h deleted file mode 100644 index 39eca5b..0000000 --- a/cpp/lubuntuci_lib.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (C) 2024-2025 Simon Quigley -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -#ifndef LUBUNTUCI_LIB_H -#define LUBUNTUCI_LIB_H - -#include -#include -#include "ci_logic.h" - -class LubuntuCI { -public: - /** - * List all known repositories from the merged config. - */ - std::vector> list_known_repos(int page = 0, - int per_page = 0, - const std::string& sort_by = "", - const std::string& sort_order = ""); - - /** - * Pull a specific repository by name (returns true on success). - */ - bool pull_repo(const std::string &repo_name, std::shared_ptr log = NULL); - - bool create_project_tarball(const std::string &repo_name, std::shared_ptr log); - - /** - * Build a specific repository by name (returns true on success). - */ - bool build_repo(const std::string &repo_name, std::shared_ptr log = NULL); - - CiLogic cilogic = CiLogic(); -}; - -#endif // LUBUNTUCI_LIB_H diff --git a/cpp/task_queue.cpp b/cpp/task_queue.cpp index a998367..adab190 100644 --- a/cpp/task_queue.cpp +++ b/cpp/task_queue.cpp @@ -33,22 +33,16 @@ void TaskQueue::enqueue(std::shared_ptr jobstatus, auto now = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); - - // Create the task std::shared_ptr task_ptr = std::make_shared(jobstatus, now, packageconf); - task_ptr->func = [task_func, self_weak = std::weak_ptr(task_ptr)](std::shared_ptr log) { - std::shared_ptr task_locked = self_weak.lock(); - if (task_locked) { - log->assign_task_context(task_locked); + task_ptr->func = [task_func, self_weak = std::weak_ptr(task_ptr)](std::shared_ptr log) mutable { + if (auto task_locked = self_weak.lock()) task_func(log); - } }; packageconf->assign_task(jobstatus, task_ptr, packageconf); - std::unique_lock lock(tasks_mutex_); tasks_.emplace(task_ptr); } - cv_.notify_all(); // Notify worker threads + cv_.notify_all(); } void TaskQueue::start() { @@ -59,14 +53,12 @@ void TaskQueue::start() { } void TaskQueue::stop() { - { - std::unique_lock tasks_lock(tasks_mutex_); - std::unique_lock packages_lock(running_packages_mutex_); - std::unique_lock running_tasks_lock(running_tasks_mutex_); + { + std::unique_lock lock(tasks_mutex_); stop_ = true; } - cv_.notify_all(); // Wake up all threads - for (auto& worker : workers_) { + cv_.notify_all(); + for (auto &worker : workers_) { if (worker.joinable()) { worker.join(); } @@ -88,63 +80,42 @@ void TaskQueue::worker_thread() { while (true) { std::shared_ptr task_to_execute; { - std::lock_guard tasks_lock(tasks_mutex_); + std::unique_lock lock(tasks_mutex_); + cv_.wait(lock, [this] { return stop_ || !tasks_.empty(); }); if (stop_ && tasks_.empty()) return; - auto it = tasks_.begin(); - bool found_valid = false; - // Iterate through the set until a valid task is found while (it != tasks_.end()) { + int package_id = (*it)->get_parent_packageconf()->package->id; { - std::shared_ptr it_task = *it; - task_to_execute = it_task; - } - - int package_id = task_to_execute->get_parent_packageconf()->package->id; - - { - std::lock_guard lock(running_packages_mutex_); - auto running_package_it = std::find_if(running_packages_.begin(), running_packages_.end(), - [&package_id](const std::shared_ptr& package) { return package->id == package_id; }); - - if (running_package_it != running_packages_.end()) { - ++it; // Move to the next task + std::lock_guard pkg_lock(running_packages_mutex_); + auto running_it = std::find_if(running_packages_.begin(), running_packages_.end(), + [package_id](const std::shared_ptr &pkg) { return pkg->id == package_id; }); + if (running_it != running_packages_.end()) { + ++it; continue; } } - - // Task is valid to execute - found_valid = true; - it = tasks_.erase(it); + task_to_execute = *it; + tasks_.erase(it); break; } - if (!found_valid) { continue; } } - if (!task_to_execute || !task_to_execute->func) continue; - else { - { - std::lock_guard packages_lock(running_packages_mutex_); - running_packages_.insert(task_to_execute->get_parent_packageconf()->package); - } - { - std::lock_guard tasks_lock(running_tasks_mutex_); - running_tasks_.insert(task_to_execute); - } + { + std::lock_guard pkg_lock(running_packages_mutex_); + running_packages_.insert(task_to_execute->get_parent_packageconf()->package); } - - // Set the start time { - auto now = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); - task_to_execute->start_time = now; + std::lock_guard rt_lock(running_tasks_mutex_); + running_tasks_.insert(task_to_execute); } - + auto now = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + task_to_execute->start_time = now; try { - task_to_execute->func(task_to_execute->log); // Execute the task + task_to_execute->func(task_to_execute->log); task_to_execute->successful = true; - } catch (const std::exception& e) { + } catch (const std::exception &e) { task_to_execute->successful = false; std::ostringstream oss; oss << "Exception type: " << typeid(e).name() << "\n" @@ -154,39 +125,27 @@ void TaskQueue::worker_thread() { task_to_execute->successful = false; task_to_execute->log->append("Unknown exception occurred"); } - + now = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + task_to_execute->finish_time = now; { - auto now = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); - task_to_execute->finish_time = now; - } - - { - // Remove the task from running_tasks_ - std::lock_guard lock(running_tasks_mutex_); + std::lock_guard rt_lock(running_tasks_mutex_); int id = task_to_execute->id; - auto running_task_it = std::find_if(running_tasks_.begin(), running_tasks_.end(), - [&id](const std::shared_ptr& task) { return task->id == id; }); - - if (running_task_it != running_tasks_.end()) { - running_tasks_.erase(running_task_it); + auto it = std::find_if(running_tasks_.begin(), running_tasks_.end(), + [id](const std::shared_ptr &task) { return task->id == id; }); + if (it != running_tasks_.end()) { + running_tasks_.erase(it); } } - { - // Remove packageconf from running_packages_ by id - std::lock_guard lock(running_packages_mutex_); + std::lock_guard pkg_lock(running_packages_mutex_); int package_id = task_to_execute->get_parent_packageconf()->package->id; - auto running_package_it = std::find_if(running_packages_.begin(), running_packages_.end(), - [&package_id](const std::shared_ptr& package) { return package->id == package_id; }); - - if (running_package_it != running_packages_.end()) { - running_packages_.erase(running_package_it); + auto it = std::find_if(running_packages_.begin(), running_packages_.end(), + [package_id](const std::shared_ptr &pkg) { return pkg->id == package_id; }); + if (it != running_packages_.end()) { + running_packages_.erase(it); } } - - // Save to the database at the end task_to_execute->save(0); } } diff --git a/cpp/update-maintainer.cpp b/cpp/update-maintainer.cpp index ea919e7..953f7ff 100644 --- a/cpp/update-maintainer.cpp +++ b/cpp/update-maintainer.cpp @@ -1,6 +1,4 @@ -// cpp/update-maintainer.cpp - -// Copyright (C) 2024 Simon Quigley +// Copyright (C) 2024-2025 Simon Quigley // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -15,7 +13,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -#include "lubuntuci_lib.h" +#include "ci_logic.h" #include int main(int argc, char** argv) { diff --git a/cpp/utilities.cpp b/cpp/utilities.cpp index ea74915..acdd981 100644 --- a/cpp/utilities.cpp +++ b/cpp/utilities.cpp @@ -14,8 +14,9 @@ // along with this program. If not, see . #include "utilities.h" -#include "common.h" +#include "/usr/include/archive.h" +#include #include #include #include @@ -25,17 +26,16 @@ #include #include #include -#include #include -#include // for std::format in C++20/23 +#include +#include -namespace fs = std::filesystem; +bool verbose = false; // Define a semaphore with a maximum of 10 concurrent jobs static std::counting_semaphore<10> sem(10); // Job queue and synchronization primitives -static std::queue> job_queue; static std::mutex queue_mutex; static std::atomic daemon_running{false}; @@ -296,3 +296,308 @@ void run_task_every(std::stop_token _stop_token, int interval_minutes, std::func std::this_thread::sleep_for(std::chrono::minutes(interval_minutes)); } } + +// Logger function implementations +void log_info(const std::string &msg) { + std::cout << "[INFO] " << msg << "\n"; +} + +void log_warning(const std::string &msg) { + std::cerr << "[WARNING] " << msg << "\n"; +} + +void log_error(const std::string &msg) { + std::cerr << "[ERROR] " << msg << "\n"; +} + +void log_verbose(const std::string &msg) { + if (verbose) { + std::cout << "[VERBOSE] " << msg << "\n"; + } +} + +namespace fs = std::filesystem; + +bool run_command(const std::vector &cmd, + const std::optional &cwd, + bool show_output, + std::shared_ptr log) { + if (cmd.empty()) { + throw std::runtime_error("Command is empty"); + } + + QProcess process; + + // Set the working directory if provided + if (cwd) { + process.setWorkingDirectory(QString::fromStdString(cwd->string())); + } + + // Set up the environment (if needed) + QProcessEnvironment env = QProcessEnvironment::systemEnvironment(); + process.setProcessEnvironment(env); + + // Extract executable and arguments + QString program = QString::fromStdString(cmd[0]); + QStringList arguments; + for (size_t i = 1; i < cmd.size(); ++i) { + arguments << QString::fromStdString(cmd[i]); + } + + // Start the command + process.start(program, arguments); + if (!process.waitForStarted()) { + throw std::runtime_error("Failed to start the command: " + program.toStdString()); + } + + // Stream output while the process is running + while (process.state() == QProcess::Running) { + if (process.waitForReadyRead()) { + QByteArray output = process.readAllStandardOutput(); + QByteArray error = process.readAllStandardError(); + + if (log) { + log->append(output.toStdString()); + log->append(error.toStdString()); + } + + if (show_output) { + std::cout << output.toStdString(); + std::cerr << error.toStdString(); + } + } + } + + // Wait for the process to finish + process.waitForFinished(); + + // Capture return code and errors + if (process.exitStatus() != QProcess::NormalExit || process.exitCode() != 0) { + QByteArray error_output = process.readAllStandardError(); + std::string error_message = "Command failed with exit code: " + std::to_string(process.exitCode()); + if (!error_output.isEmpty()) { + error_message += "\nError Output: " + error_output.toStdString(); + } + throw std::runtime_error(error_message); + } + + return true; +} + +// Function to extract excluded files from a copyright file +std::vector extract_files_excluded(const std::string& filepath) { + std::ifstream file(filepath); + if (!file.is_open()) { + throw std::runtime_error("Failed to open file: " + filepath); + } + + std::vector files_excluded; + std::string line; + std::regex files_excluded_pattern(R"(Files-Excluded:\s*(.*))"); + bool in_files_excluded = false; + + while (std::getline(file, line)) { + if (std::regex_match(line, files_excluded_pattern)) { + in_files_excluded = true; + std::smatch match; + if (std::regex_search(line, match, files_excluded_pattern) && match.size() > 1) { + files_excluded.emplace_back(match[1]); + } + } else if (in_files_excluded) { + if (!line.empty() && (line[0] == ' ' || line[0] == '\t')) { + files_excluded.emplace_back(line.substr(1)); + } else { + break; // End of Files-Excluded block + } + } + } + + return files_excluded; +} + +// Function to create a tarball +void create_tarball(const std::string& tarballPath, const std::string& directory, const std::vector& exclusions, std::shared_ptr log) { + log->append("Creating tarball: " + tarballPath); + + struct archive* a = archive_write_new(); + if (!a) { + throw std::runtime_error("Failed to create a new archive."); + } + + if (archive_write_add_filter_gzip(a) != ARCHIVE_OK) { + std::string err = "Failed to add gzip filter: "; + err += archive_error_string(a); + archive_write_free(a); + throw std::runtime_error(err); + } + + if (archive_write_set_format_pax_restricted(a) != ARCHIVE_OK) { + std::string err = "Failed to set format: "; + err += archive_error_string(a); + archive_write_free(a); + throw std::runtime_error(err); + } + + if (archive_write_open_filename(a, tarballPath.c_str()) != ARCHIVE_OK) { + std::string err = "Could not open tarball for writing: "; + err += archive_error_string(a); + archive_write_free(a); + throw std::runtime_error(err); + } + + // Initialize a set to track added relative paths to prevent duplication + std::unordered_set added_paths; + + // Iterate through the directory recursively without following symlinks + for (auto it = fs::recursive_directory_iterator( + directory, + fs::directory_options::skip_permission_denied); + it != fs::recursive_directory_iterator(); ++it) { + const auto& path = it->path(); + std::error_code ec; + + fs::path relative_path = fs::relative(path, directory, ec); + if (ec) { + log->append("Failed to compute relative path for: " + path.string() + " Error: " + ec.message()); + continue; + } + + // Normalize the relative path to avoid discrepancies + fs::path normalized_relative_path = relative_path.lexically_normal(); + std::string relative_path_str = normalized_relative_path.string(); + + // Check if this path has already been added + if (!added_paths.insert(relative_path_str).second) { + log->append("Duplicate path detected and skipped: " + relative_path_str); + continue; // Skip adding this duplicate path + } + + // Exclusion logic (if any exclusions are provided) + bool excluded = std::any_of(exclusions.begin(), exclusions.end(), [&relative_path_str](const std::string& exclusion) { + return relative_path_str.find(exclusion) != std::string::npos; + }); + if (excluded) { continue; } + + fs::file_status fstatus = it->symlink_status(ec); + if (ec) { + log->append("Failed to get file status for: " + path.string() + " Error: " + ec.message()); + continue; + } + + struct archive_entry* entry = archive_entry_new(); + if (!entry) { + log->append("Failed to create archive entry for: " + path.string()); + archive_write_free(a); + throw std::runtime_error("Failed to create archive entry."); + } + + std::string entry_path = relative_path_str; + if (fs::is_directory(fstatus)) { + // Ensure the directory pathname ends with '/' + if (!entry_path.empty() && entry_path.back() != '/') { + entry_path += '/'; + } + archive_entry_set_pathname(entry, entry_path.c_str()); + } else { + archive_entry_set_pathname(entry, entry_path.c_str()); + } + + // Set file type, permissions, and size + if (fs::is_regular_file(fstatus)) { + // Regular file + uintmax_t filesize = fs::file_size(path, ec); + if (ec) { + log->append("Cannot get file size for: " + path.string() + " Error: " + ec.message()); + archive_entry_free(entry); + continue; + } + archive_entry_set_size(entry, static_cast(filesize)); + archive_entry_set_filetype(entry, AE_IFREG); + archive_entry_set_perm(entry, static_cast(fstatus.permissions())); + } + else if (fs::is_symlink(fstatus)) { + fs::path target = fs::read_symlink(path, ec); + if (ec) { + log->append("Cannot read symlink for: " + path.string() + " Error: " + ec.message()); + archive_entry_free(entry); + continue; + } + archive_entry_set_symlink(entry, target.c_str()); + archive_entry_set_filetype(entry, AE_IFLNK); + archive_entry_set_perm(entry, static_cast(fstatus.permissions())); + } + else if (fs::is_directory(fstatus)) { + archive_entry_set_size(entry, 0); + archive_entry_set_filetype(entry, AE_IFDIR); + archive_entry_set_perm(entry, static_cast(fstatus.permissions())); + } + else { + log->append("Unsupported file type for: " + path.string()); + archive_entry_free(entry); + continue; + } + + // Retrieve and set the modification time + fs::file_time_type ftime = fs::last_write_time(path, ec); + std::time_t mtime; + if (ec) { + log->append("Failed to get last write time for: " + path.string() + " Error: " + ec.message()); + // Obtain current UTC time as fallback + auto now = std::chrono::system_clock::now(); + mtime = std::chrono::system_clock::to_time_t(now); + log->append("Setting default mtime (current UTC time) for: " + path.string()); + } else { + mtime = to_time_t(ftime); + } + archive_entry_set_mtime(entry, mtime, 0); + + if (archive_write_header(a, entry) != ARCHIVE_OK) { + log->append("Failed to write header for: " + path.string() + " Error: " + archive_error_string(a)); + archive_entry_free(entry); + continue; + } + + if (fs::is_regular_file(fstatus)) { + std::ifstream fileStream(path, std::ios::binary); + if (!fileStream) { + log->append("Failed to open file for reading: " + path.string()); + archive_entry_free(entry); + continue; + } + + const std::size_t bufferSize = 8192; + char buffer[bufferSize]; + while (fileStream) { + fileStream.read(buffer, bufferSize); + std::streamsize bytesRead = fileStream.gcount(); + if (bytesRead > 0) { + if (archive_write_data(a, buffer, static_cast(bytesRead)) < 0) { + log->append("Failed to write data for: " + path.string() + " Error: " + archive_error_string(a)); + break; + } + } + } + + if (fileStream.bad()) { + log->append("Error reading file: " + path.string()); + } + } + + archive_entry_free(entry); + } + + if (archive_write_close(a) != ARCHIVE_OK) { + std::string err = "Failed to close archive: "; + err += archive_error_string(a); + archive_write_free(a); + throw std::runtime_error(err); + } + + if (archive_write_free(a) != ARCHIVE_OK) { + std::string err = "Failed to free archive: "; + err += archive_error_string(a); + throw std::runtime_error(err); + } + + log->append("Tarball created and compressed: " + tarballPath); +} diff --git a/cpp/utilities.h b/cpp/utilities.h index 5ff535c..10936ef 100644 --- a/cpp/utilities.h +++ b/cpp/utilities.h @@ -18,11 +18,57 @@ #include #include #include +#include #include +#include #include #include - #include +#include + +namespace fs = std::filesystem; + +class Task; + +// Time utilities +std::string get_current_utc_time(const std::string& format); +std::time_t to_time_t(const std::filesystem::file_time_type& ftime); + +class Log { +private: + std::string data = ""; + mutable std::shared_mutex lock_; + std::weak_ptr task_context_; + std::string last_data_str = ""; + +public: + void append(const std::string& str) { + std::unique_lock lock(lock_); + std::string log_str = str.ends_with('\n') ? str : str + '\n'; + if (str.empty() || last_data_str == log_str) { return; } + else if (str.contains("dpkg-source: warning: ignoring deletion of file")) { return; } + data += std::format("[{}] {}", get_current_utc_time("%Y-%m-%dT%H:%M:%SZ"), log_str); + last_data_str = log_str; + } + + void set_log(const std::string& str) { + std::unique_lock lock(lock_); + data = str; + } + + std::string get() const { + std::shared_lock lock(lock_); + return std::regex_replace(data, std::regex(R"(^\s+)"), ""); + } + + void assign_task_context(std::shared_ptr task) { + task_context_ = task; + } + + std::shared_ptr get_task_context() const { + return task_context_.lock(); + } +}; // Function to read the entire content of a file into a string std::string read_file(const std::filesystem::path& filePath); @@ -49,10 +95,6 @@ std::filesystem::path create_temp_directory(); // Function to copy a directory recursively void copy_directory(const std::filesystem::path& source, const std::filesystem::path& destination); -// Time utilities -std::string get_current_utc_time(const std::string& format); -std::time_t to_time_t(const std::filesystem::file_time_type& ftime); - // String utilities std::vector split_string(const std::string& input, const std::string& delimiter); std::string remove_suffix(const std::string& input, const std::string& suffix); @@ -65,3 +107,25 @@ std::pair get_version_from_codename(const std::string& codename); void ensure_git_inited(); void run_task_every(std::stop_token _stop_token, int interval_minutes, std::function task); + +// Logger functions +extern bool verbose; +void log_info(const std::string &msg); +void log_warning(const std::string &msg); +void log_error(const std::string &msg); +void log_verbose(const std::string &msg); + +// Function to run a command with optional working directory and show output +bool run_command(const std::vector &cmd, + const std::optional &cwd = std::nullopt, + bool show_output = false, + std::shared_ptr log = nullptr); + +// Function to extract excluded files from a copyright file +std::vector extract_files_excluded(const std::string& filepath); + +// Function to create a tarball +void create_tarball(const std::string& tarballPath, + const std::string& directory, + const std::vector& exclusions, + std::shared_ptr log = nullptr); diff --git a/cpp/web_server.cpp b/cpp/web_server.cpp index 0e9a56b..2643100 100644 --- a/cpp/web_server.cpp +++ b/cpp/web_server.cpp @@ -18,6 +18,8 @@ #include "sources_parser.h" #include "naive_bayes_classifier.h" #include "db_common.h" +#include "template_renderer.h" +#include "ci_logic.h" // Qt includes #include @@ -57,10 +59,6 @@ #include "build.h" #include "binary_package_publishing_history.h" -// Local includes -#include "lubuntuci_lib.h" -#include "template_renderer.h" - constexpr QHttpServerResponder::StatusCode StatusCodeFound = QHttpServerResponder::StatusCode::Found; static std::string timestamp_now() @@ -131,7 +129,8 @@ WebServer::WebServer(QObject *parent) : QObject(parent) {} QHttpServerResponse bad_response(StatusCodeFound); QHttpHeaders bad_response_headers; - bad_response_headers.replaceOrAppend(QHttpHeaders::WellKnownHeader::Location, "/unauthorized?base_url=" + base_url + "&redirect_to=" + current_path); + bad_response_headers.replaceOrAppend(QHttpHeaders::WellKnownHeader::Location, + "/unauthorized?base_url=" + base_url + "&redirect_to=" + current_path); bad_response.setHeaders(bad_response_headers); return bad_response; @@ -174,14 +173,14 @@ bool WebServer::start_server(quint16 port) { } archive proposed = proposed_opt.value(); - std::shared_ptr _tmp_pkg_conf = std::make_shared(); - std::shared_ptr lubuntuci = std::make_shared(); - std::vector> all_repos = lubuntuci->list_known_repos(); + // Use our new list_known_repos() method from CiLogic + std::shared_ptr cilogic = std::make_shared(); + std::vector> all_repos = cilogic->list_known_repos(); task_queue = std::make_unique(6); - std::shared_ptr>> job_statuses = lubuntuci->cilogic.get_job_statuses(); + std::shared_ptr>> job_statuses = cilogic->get_job_statuses(); task_queue->start(); - // Load initial tokens + // Load initial tokens from the database { QSqlQuery load_tokens(get_thread_connection()); load_tokens.prepare("SELECT person.id, person.username, person.logo_url, person_token.token, person_token.expiry_date FROM person INNER JOIN person_token ON person.id = person_token.person_id"); @@ -199,12 +198,12 @@ bool WebServer::start_server(quint16 port) { } } - expire_tokens_thread_ = std::jthread(run_task_every, 60, [this, lubuntuci] { + expire_tokens_thread_ = std::jthread(run_task_every, 60, [this, cilogic] { QSqlQuery expired_tokens(get_thread_connection()); QString current_time = QDateTime::currentDateTime().toString(Qt::ISODate); expired_tokens.prepare("DELETE FROM person_token WHERE expiry_date < :current_time"); - expired_tokens.bindValue(":current_time", QDateTime::currentDateTime().toString(Qt::ISODate)); + expired_tokens.bindValue(":current_time", current_time); ci_query_exec(&expired_tokens); for (auto it = _active_tokens.begin(); it != _active_tokens.end();) { if (it.value() <= QDateTime::currentDateTime()) it = _active_tokens.erase(it); @@ -216,10 +215,9 @@ bool WebServer::start_server(quint16 port) { } }); - process_sources_thread_ = std::jthread(run_task_every, 10, [this, all_repos, proposed, lubuntuci, job_statuses] { + process_sources_thread_ = std::jthread(run_task_every, 10, [this, all_repos, proposed, cilogic, job_statuses] { for (auto pkgconf : all_repos) { - if (!pkgconf->can_check_source_upload()) { continue; } - + if (!pkgconf->can_check_source_upload()) continue; task_queue->enqueue( job_statuses->at("source_check"), [this, pkgconf, proposed](std::shared_ptr log) mutable { @@ -229,22 +227,17 @@ bool WebServer::start_server(quint16 port) { found_in_ppa = true; break; } - - if (!found_in_ppa) { - throw std::runtime_error("Not found in the PPA."); - } + if (!found_in_ppa) throw std::runtime_error("Not found in the PPA."); }, pkgconf ); - pkgconf->sync(); } }); - process_binaries_thread_ = std::jthread(run_task_every, 15, [this, all_repos, proposed, lubuntuci, job_statuses] { + process_binaries_thread_ = std::jthread(run_task_every, 15, [this, all_repos, proposed, cilogic, job_statuses] { for (auto pkgconf : all_repos) { - if (!pkgconf->can_check_builds()) { continue; } - + if (!pkgconf->can_check_builds()) continue; task_queue->enqueue( job_statuses->at("build_check"), [this, pkgconf, proposed](std::shared_ptr log) mutable { @@ -279,7 +272,7 @@ bool WebServer::start_server(quint16 port) { //////////////////////////////////////////////////////////////// // /unauthorized?base_url=&redirect_to= //////////////////////////////////////////////////////////////// - http_server_.route("/unauthorized", [this, lubuntuci](const QHttpServerRequest &req) -> QFuture { + http_server_.route("/unauthorized", [this, cilogic](const QHttpServerRequest &req) -> QFuture { // Extract data up front auto query = req.query(); QString base_url = query.queryItemValue("base_url"); @@ -335,7 +328,7 @@ bool WebServer::start_server(quint16 port) { ///////////////// // /authcallback ///////////////// - http_server_.route("/authcallback", [this, lubuntuci](const QHttpServerRequest &req) -> QFuture { + http_server_.route("/authcallback", [this, cilogic](const QHttpServerRequest &req) -> QFuture { // Extract data up front auto query = req.query(); QString base_url = query.queryItemValue("base_url"); @@ -476,7 +469,7 @@ bool WebServer::start_server(quint16 port) { ////////////////////////////////////////// // Route "/" ////////////////////////////////////////// - http_server_.route("/", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture { + http_server_.route("/", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture { { QHttpServerResponse session_response = verify_session_token(req, req.headers()); if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); }); @@ -492,13 +485,13 @@ bool WebServer::start_server(quint16 port) { : query.queryItemValue("sort_order").toStdString(); return QtConcurrent::run([=, this]() { - auto all_repos = lubuntuci->list_known_repos(); + auto all_repos = cilogic->list_known_repos(); int total_size = static_cast(all_repos.size()); int total_pages = (per_page > 0) ? (total_size + per_page - 1) / per_page : 1; - auto repos = lubuntuci->list_known_repos(page, per_page, sort_by, sort_order); + auto repos = cilogic->list_known_repos(page, per_page, sort_by, sort_order); if (repos.empty() && total_size == 0) { std::string err_html = R"( @@ -519,37 +512,31 @@ bool WebServer::start_server(quint16 port) { {"total_pages", std::to_string(total_pages)} }; std::map>> list_context; - std::vector> reposVec; for (const auto &r : repos) { std::map item; std::string packaging_commit_str; std::string upstream_commit_str; - if (r->packaging_commit) { std::string commit_summary = r->packaging_commit->commit_summary; - if (commit_summary.size() > 40) { + if (commit_summary.size() > 40) commit_summary = commit_summary.substr(0, 37) + "..."; - } packaging_commit_str = r->packaging_commit->commit_hash.substr(0, 7) + std::format(" ({:%Y-%m-%d %H:%M:%S %Z})
", r->packaging_commit->commit_datetime) + commit_summary; } if (r->upstream_commit) { std::string commit_summary = r->upstream_commit->commit_summary; - if (commit_summary.size() > 40) { + if (commit_summary.size() > 40) commit_summary = commit_summary.substr(0, 37) + "..."; - } upstream_commit_str = r->upstream_commit->commit_hash.substr(0, 7) + std::format(" ({:%Y-%m-%d %H:%M:%S %Z})
", r->upstream_commit->commit_datetime) + commit_summary; } - std::string packaging_commit_url_str = (r->package ? r->package->packaging_browser : "") + (r->packaging_commit ? r->packaging_commit->commit_hash : ""); std::string upstream_commit_url_str = (r->package ? r->package->upstream_browser : "") + (r->upstream_commit ? r->upstream_commit->commit_hash : ""); - item["id"] = std::to_string(r->id); item["name"] = r->package->name; item["branch_name"] = r->branch->name; @@ -597,7 +584,7 @@ bool WebServer::start_server(quint16 port) { ////////////////////////////////////////// // /pull?repo= ////////////////////////////////////////// - http_server_.route("/pull", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture { + http_server_.route("/pull", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture { { QHttpServerResponse session_response = verify_session_token(req, req.headers()); if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); }); @@ -612,7 +599,7 @@ bool WebServer::start_server(quint16 port) { } int repo = std::stoi(repo_string.toStdString()); - std::string msg = lubuntuci->cilogic.queue_pull_tarball({ lubuntuci->cilogic.get_packageconf_by_id(repo) }, task_queue, job_statuses); + std::string msg = cilogic->queue_pull_tarball({ cilogic->get_packageconf_by_id(repo) }, task_queue, job_statuses); return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size())); }); }); @@ -620,7 +607,7 @@ bool WebServer::start_server(quint16 port) { ////////////////////////////////////////// // /build?repo= ////////////////////////////////////////// - http_server_.route("/build", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture { + http_server_.route("/build", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture { { QHttpServerResponse session_response = verify_session_token(req, req.headers()); if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); }); @@ -635,7 +622,7 @@ bool WebServer::start_server(quint16 port) { } int repo = std::stoi(repo_string.toStdString()); - std::string msg = lubuntuci->cilogic.queue_build_upload({ lubuntuci->cilogic.get_packageconf_by_id(repo) }, task_queue, job_statuses); + std::string msg = cilogic->queue_build_upload({ cilogic->get_packageconf_by_id(repo) }, task_queue, job_statuses); return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size())); }); }); @@ -643,7 +630,7 @@ bool WebServer::start_server(quint16 port) { ////////////////////////////////////////// // /pull-selected?repos= ////////////////////////////////////////// - http_server_.route("/pull-selected", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture { + http_server_.route("/pull-selected", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture { { QHttpServerResponse session_response = verify_session_token(req, req.headers()); if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); }); @@ -667,7 +654,7 @@ bool WebServer::start_server(quint16 port) { }) ); - std::string msg = lubuntuci->cilogic.queue_pull_tarball(lubuntuci->cilogic.get_packageconfs_by_ids(repos), task_queue, job_statuses); + std::string msg = cilogic->queue_pull_tarball(cilogic->get_packageconfs_by_ids(repos), task_queue, job_statuses); return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size())); }); }); @@ -675,7 +662,7 @@ bool WebServer::start_server(quint16 port) { ////////////////////////////////////////// // /build-selected?repos=foo,bar,baz ////////////////////////////////////////// - http_server_.route("/build-selected", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture { + http_server_.route("/build-selected", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture { { QHttpServerResponse session_response = verify_session_token(req, req.headers()); if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); }); @@ -699,7 +686,7 @@ bool WebServer::start_server(quint16 port) { }) ); - std::string msg = lubuntuci->cilogic.queue_build_upload(lubuntuci->cilogic.get_packageconfs_by_ids(repos), task_queue, job_statuses); + std::string msg = cilogic->queue_build_upload(cilogic->get_packageconfs_by_ids(repos), task_queue, job_statuses); return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size())); }); }); @@ -707,7 +694,7 @@ bool WebServer::start_server(quint16 port) { ////////////////////////////////////////// // /pull-and-build-selected?repos=foo,bar,baz ////////////////////////////////////////// - http_server_.route("/pull-and-build-selected", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture { + http_server_.route("/pull-and-build-selected", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture { { QHttpServerResponse session_response = verify_session_token(req, req.headers()); if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); }); @@ -730,11 +717,11 @@ bool WebServer::start_server(quint16 port) { return std::stoi(s); }) ); - auto pkgconfs = lubuntuci->cilogic.get_packageconfs_by_ids(repos); + auto pkgconfs = cilogic->get_packageconfs_by_ids(repos); for (auto pkgconf : pkgconfs) pkgconf->clear_tasks(); - std::string msg = lubuntuci->cilogic.queue_pull_tarball(pkgconfs, task_queue, job_statuses); - msg += lubuntuci->cilogic.queue_build_upload(pkgconfs, task_queue, job_statuses); + std::string msg = cilogic->queue_pull_tarball(pkgconfs, task_queue, job_statuses); + msg += cilogic->queue_build_upload(pkgconfs, task_queue, job_statuses); return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size())); }); }); @@ -742,13 +729,13 @@ bool WebServer::start_server(quint16 port) { ////////////////////////////////////////// // /pull-all ////////////////////////////////////////// - http_server_.route("/pull-all", [this, lubuntuci, all_repos, job_statuses](const QHttpServerRequest &req) -> QFuture { + http_server_.route("/pull-all", [this, cilogic, all_repos, job_statuses](const QHttpServerRequest &req) -> QFuture { { QHttpServerResponse session_response = verify_session_token(req, req.headers()); if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); }); } return QtConcurrent::run([=, this]() { - std::string msg = lubuntuci->cilogic.queue_pull_tarball(all_repos, task_queue, job_statuses); + std::string msg = cilogic->queue_pull_tarball(all_repos, task_queue, job_statuses); return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size())); }); @@ -757,13 +744,13 @@ bool WebServer::start_server(quint16 port) { ////////////////////////////////////////// // /build-all ////////////////////////////////////////// - http_server_.route("/build-all", [this, lubuntuci, all_repos, job_statuses](const QHttpServerRequest &req) -> QFuture { + http_server_.route("/build-all", [this, cilogic, all_repos, job_statuses](const QHttpServerRequest &req) -> QFuture { { QHttpServerResponse session_response = verify_session_token(req, req.headers()); if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); }); } return QtConcurrent::run([=, this]() { - std::string msg = lubuntuci->cilogic.queue_build_upload(all_repos, task_queue, job_statuses); + std::string msg = cilogic->queue_build_upload(all_repos, task_queue, job_statuses); return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size())); }); @@ -772,15 +759,15 @@ bool WebServer::start_server(quint16 port) { ////////////////////////////////////////// // /pull-and-build-all ////////////////////////////////////////// - http_server_.route("/pull-and-build-all", [this, lubuntuci, all_repos, job_statuses](const QHttpServerRequest &req) -> QFuture { + http_server_.route("/pull-and-build-all", [this, cilogic, all_repos, job_statuses](const QHttpServerRequest &req) -> QFuture { { QHttpServerResponse session_response = verify_session_token(req, req.headers()); if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); }); } return QtConcurrent::run([=, this]() { for (auto pkgconf : all_repos) pkgconf->clear_tasks(); - std::string msg = lubuntuci->cilogic.queue_pull_tarball(all_repos, task_queue, job_statuses); - msg += lubuntuci->cilogic.queue_build_upload(all_repos, task_queue, job_statuses); + std::string msg = cilogic->queue_pull_tarball(all_repos, task_queue, job_statuses); + msg += cilogic->queue_build_upload(all_repos, task_queue, job_statuses); return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size())); }); @@ -789,7 +776,7 @@ bool WebServer::start_server(quint16 port) { ////////////////////////////////////////// // Serve static files from /static/ ////////////////////////////////////////// - http_server_.route("/static/", [this, lubuntuci, job_statuses](const QString filename) -> QHttpServerResponse { + http_server_.route("/static/", [this, cilogic, job_statuses](const QString filename) -> QHttpServerResponse { QString sanitized_filename = filename; if (filename.contains("..") || filename.contains("../")) { return QHttpServerResponse(QHttpServerResponder::StatusCode::BadRequest); @@ -827,7 +814,7 @@ bool WebServer::start_server(quint16 port) { ////////////////////////////////////////// // /graph ////////////////////////////////////////// - http_server_.route("/graph", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture { + http_server_.route("/graph", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture { { QHttpServerResponse session_response = verify_session_token(req, req.headers()); if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); }); @@ -875,7 +862,7 @@ bool WebServer::start_server(quint16 port) { ////////////////////////////////////////// // /tasks ////////////////////////////////////////// - http_server_.route("/tasks", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture { + http_server_.route("/tasks", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture { { QHttpServerResponse session_response = verify_session_token(req, req.headers()); if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); }); @@ -911,7 +898,7 @@ bool WebServer::start_server(quint16 port) { title_prefix = "Completed"; // gather tasks that have start_time > 0 and finish_time > 0 std::vector> tasks_vector; - auto pkgconfs = lubuntuci->cilogic.get_packageconfs(); + auto pkgconfs = cilogic->get_packageconfs(); for (auto &pkgconf : pkgconfs) { for (auto &j : *job_statuses) { if (!j.second) { @@ -967,7 +954,7 @@ bool WebServer::start_server(quint16 port) { ////////////////////////////////////////// // /log/ ////////////////////////////////////////// - http_server_.route("/log/", [this, lubuntuci, job_statuses](const QString _task_id, const QHttpServerRequest &req) -> QFuture { + http_server_.route("/log/", [this, cilogic, job_statuses](const QString _task_id, const QHttpServerRequest &req) -> QFuture { { QHttpServerResponse session_response = verify_session_token(req, req.headers()); if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); }); @@ -984,19 +971,12 @@ bool WebServer::start_server(quint16 port) { std::string msg = "

Invalid task ID specified.

"; return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size())); } - - std::string log_content = lubuntuci->cilogic.get_task_log(task_id); - - std::map>> list_context; + std::string log_content = cilogic->get_task_log(task_id); std::map context; + std::map>> list_context; context["title"] = "Task Logs"; context["log"] = log_content; - - std::string final_html = TemplateRenderer::render_with_inheritance( - "log.html", - context, - list_context - ); + std::string final_html = TemplateRenderer::render_with_inheritance("log.html", context, list_context); return QHttpServerResponse("text/html", QByteArray(final_html.c_str(), (int)final_html.size())); }); }); @@ -1011,7 +991,6 @@ bool WebServer::start_server(quint16 port) { key_file.open(QIODevice::ReadOnly); ssl_config.setPrivateKey(QSslKey(&key_file, QSsl::Rsa, QSsl::Pem)); key_file.close(); - ssl_config.setPeerVerifyMode(QSslSocket::VerifyNone); ssl_config.setProtocol(QSsl::TlsV1_3); ssl_server_.setSslConfiguration(ssl_config);