Lubuntu/ci-tools
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Various cleanup bits

Browse Source
This commit is contained in:
Simon Quigley 2025-02-01 17:02:05 -06:00
parent 6eada27b20
commit 06a6567da9
14 changed files with 581 additions and 788 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cpp/CMakeLists.txt
View File

@ -40,14 +40,12 @@ set(UUID_LIB "uuid")
# 1. The main library: lubuntuci_lib
#
add_library(lubuntuci_lib SHARED
common.cpp
utilities.cpp
db_common.cpp
git_common.cpp
sources_parser.cpp
ci_logic.cpp
ci_database_objs.cpp
lubuntuci_lib.cpp
task_queue.cpp
template_renderer.cpp
web_server.cpp

2
cpp/ci_database_objs.h
View File

@ -27,7 +27,7 @@
#include <QSqlDatabase>
#include <yaml-cpp/yaml.h>
#include "common.h"
#include "utilities.h"
class Person {
public:

39
cpp/ci_logic.cpp
View File

@ -6,7 +6,7 @@
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// but WITHOUT ANY WARRANTY, without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
@ -15,8 +15,6 @@
#include "task_queue.h"
#include "ci_logic.h"
#include "lubuntuci_lib.h"
#include "common.h"
#include "utilities.h"
#include "db_common.h"
#include "git_common.h"
@ -530,7 +528,7 @@ void CiLogic::process_entire_pipeline(std::shared_ptr<PackageConf> &proj,
bool pull_success = pull_project(proj);
bool tarball_success = create_project_tarball(proj);
const auto [build_success, changes_files] = build_project(proj);
upload_and_lint(proj, changes_files, skip_dput);
upload_and_lint(proj, changes_files, skip_dput, std::make_shared<Log>());
do_summary(skip_cleanup);
log_info("Pipeline done for " + proj->package->name);
} catch(std::exception &ex) {
@ -829,3 +827,36 @@ std::string CiLogic::get_task_log(int task_id) {
}
return "";
}
std::vector<std::shared_ptr<PackageConf>> CiLogic::list_known_repos(int page,
int per_page,
const std::string &sort_by,
const std::string &sort_order) {
init_global();
if (page != 0 && per_page != 0 && !sort_by.empty() && !sort_order.empty()) return get_config("", page, per_page, sort_by, sort_order);
else return get_config();
}
bool CiLogic::pull_repo_by_name(const std::string &repo_name, std::shared_ptr<Log> log) {
init_global();
auto pkgconfs = get_config(repo_name);
if (pkgconfs.empty()) return false;
return pull_project(pkgconfs.at(0), log);
}
bool CiLogic::create_project_tarball_by_name(const std::string &repo_name, std::shared_ptr<Log> log) {
init_global();
auto pkgconfs = get_config(repo_name);
if (pkgconfs.empty()) return false;
return create_project_tarball(pkgconfs.at(0), log);
}
bool CiLogic::build_repo_by_name(const std::string &repo_name, std::shared_ptr<Log> log) {
init_global();
bool success = true;
for (auto pkgconf : get_config(repo_name)) {
auto [build_ok, changes_files] = build_project(pkgconf, log);
success = success && build_ok && upload_and_lint(pkgconf, changes_files, false, log);
}
return success;
}

127
cpp/ci_logic.h
View File

@ -32,11 +32,8 @@
#include <QSqlDatabase>
#include <yaml-cpp/yaml.h>
struct CiProject;
namespace fs = std::filesystem;
/**
* Data describing one package to pull/build/etc.
*/
struct CiProject {
std::string name;
std::string version;
@ -45,78 +42,98 @@ struct CiProject {
std::string upstream_url;
std::string packaging_url;
std::optional<std::string> packaging_branch;
std::filesystem::path main_tarball;
fs::path main_tarball;
bool large = false;
// These get populated during build
// These get populated during build:
std::vector<std::string> changes_files;
std::vector<std::string> devel_changes_files;
};
class CiLogic {
public:
// Initialize global configurations
void init_global();
public:
// Initialize global config and database
void init_global();
// Load YAML configuration from a given path
YAML::Node load_yaml_config(const std::filesystem::path &config_path);
// Load YAML config from a given path
YAML::Node load_yaml_config(const fs::path &config_path);
// Convert a YAML node to a CiProject structure
CiProject yaml_to_project(const YAML::Node &pkg_node);
// Convert a YAML node to a CiProject
CiProject yaml_to_project(const YAML::Node &pkg_node);
bool pull_project(std::shared_ptr<PackageConf> &proj, std::shared_ptr<Log> log = NULL);
bool create_project_tarball(std::shared_ptr<PackageConf> &proj, std::shared_ptr<Log> log = NULL);
std::tuple<bool, std::set<std::string>> build_project(std::shared_ptr<PackageConf> proj, std::shared_ptr<Log> log = NULL);
bool upload_and_lint(std::shared_ptr<PackageConf> &proj, const std::set<std::string> changes_files, bool skip_dput, std::shared_ptr<Log> log = NULL);
// Pipeline functions
bool pull_project(std::shared_ptr<PackageConf> &proj, std::shared_ptr<Log> log = nullptr);
bool create_project_tarball(std::shared_ptr<PackageConf> &proj, std::shared_ptr<Log> log = nullptr);
std::tuple<bool, std::set<std::string>> build_project(std::shared_ptr<PackageConf> proj, std::shared_ptr<Log> log = nullptr);
bool upload_and_lint(std::shared_ptr<PackageConf> &proj,
const std::set<std::string> changes_files,
bool skip_dput,
std::shared_ptr<Log> log = nullptr);
// Perform cleanup and summarize the build process
void do_summary(bool skip_cleanup);
// Summary & cleanup
void do_summary(bool skip_cleanup);
// Process the entire pipeline for a given PackageConf ID
void process_entire_pipeline(std::shared_ptr<PackageConf> &proj, bool skip_dput, bool skip_cleanup);
// Orchestrate entire pipeline
void process_entire_pipeline(std::shared_ptr<PackageConf> &proj,
bool skip_dput,
bool skip_cleanup);
// Retrieve all PackageConf entries from the database
std::vector<std::shared_ptr<PackageConf>> get_config(const std::string &repo_name = "", int page = 0, int per_page = 0, const std::string& sort_by = "", const std::string& sort_order = "");
// Retrieve PackageConf entries (with optional pagination/sorting)
std::vector<std::shared_ptr<PackageConf>> get_config(const std::string &repo_name = "",
int page = 0,
int per_page = 0,
const std::string &sort_by = "",
const std::string &sort_order = "");
// Function to enqueue tasks
void enqueue(std::function<void()> task);
// Enqueue a task (wrapper)
void enqueue(std::function<void()> task);
std::shared_ptr<std::map<std::string, std::shared_ptr<JobStatus>>> get_job_statuses();
std::vector<std::shared_ptr<PackageConf>> get_packageconfs();
std::shared_ptr<PackageConf> get_packageconf_by_id(int id);
std::vector<std::shared_ptr<PackageConf>> get_packageconfs_by_ids(std::set<int> ids);
void set_packageconfs(std::vector<std::shared_ptr<PackageConf>> _pkgconfs);
void sync(std::shared_ptr<PackageConf> pkgconf);
// Job status and PackageConf getters
std::shared_ptr<std::map<std::string, std::shared_ptr<JobStatus>>> get_job_statuses();
std::vector<std::shared_ptr<PackageConf>> get_packageconfs();
std::shared_ptr<PackageConf> get_packageconf_by_id(int id);
std::vector<std::shared_ptr<PackageConf>> get_packageconfs_by_ids(std::set<int> ids);
void set_packageconfs(std::vector<std::shared_ptr<PackageConf>> _pkgconfs);
void sync(std::shared_ptr<PackageConf> pkgconf);
std::string queue_pull_tarball(std::vector<std::shared_ptr<PackageConf>> repos,
std::unique_ptr<TaskQueue>& task_queue,
std::shared_ptr<std::map<std::string, std::shared_ptr<JobStatus>>> job_statuses);
std::string queue_build_upload(std::vector<std::shared_ptr<PackageConf>> repos,
std::unique_ptr<TaskQueue>& task_queue,
std::shared_ptr<std::map<std::string, std::shared_ptr<JobStatus>>> job_statuses);
// Queue tasks
std::string queue_pull_tarball(std::vector<std::shared_ptr<PackageConf>> repos,
std::unique_ptr<TaskQueue>& task_queue,
std::shared_ptr<std::map<std::string, std::shared_ptr<JobStatus>>> job_statuses);
std::string queue_build_upload(std::vector<std::shared_ptr<PackageConf>> repos,
std::unique_ptr<TaskQueue>& task_queue,
std::shared_ptr<std::map<std::string, std::shared_ptr<JobStatus>>> job_statuses);
std::string get_task_log(int task_id);
// Get a task’s log
std::string get_task_log(int task_id);
std::vector<Release> releases;
std::vector<Package> packages;
std::vector<Branch> branches;
std::vector<std::shared_ptr<PackageConf>> list_known_repos(int page = 0,
int per_page = 0,
const std::string &sort_by = "",
const std::string &sort_order = "");
bool pull_repo_by_name(const std::string &repo_name, std::shared_ptr<Log> log = nullptr);
bool create_project_tarball_by_name(const std::string &repo_name, std::shared_ptr<Log> log = nullptr);
bool build_repo_by_name(const std::string &repo_name, std::shared_ptr<Log> log = nullptr);
private:
void debuild_package(const fs::path &packaging_dir, std::shared_ptr<Log> log);
// These come from the config/DB
std::vector<Release> releases;
std::vector<Package> packages;
std::vector<Branch> branches;
QSqlDatabase p_db;
private:
void debuild_package(const fs::path &packaging_dir, std::shared_ptr<Log> log);
mutable std::mutex packageconfs_mutex_;
std::vector<std::shared_ptr<PackageConf>> packageconfs;
std::shared_ptr<std::map<std::string, std::shared_ptr<JobStatus>>> _cached_job_statuses;
QSqlDatabase p_db;
mutable std::mutex packageconfs_mutex_;
std::vector<std::shared_ptr<PackageConf>> packageconfs;
std::shared_ptr<std::map<std::string, std::shared_ptr<JobStatus>>> _cached_job_statuses;
struct package_conf_item {
std::shared_ptr<PackageConf> first_pkgconf;
std::shared_ptr<Task> first_pull_task = std::make_shared<Task>();
std::shared_ptr<Task> first_tarball_task = std::make_shared<Task>();
std::shared_ptr<GitCommit> packaging_commit = std::make_shared<GitCommit>();
std::shared_ptr<GitCommit> upstream_commit = std::make_shared<GitCommit>();
};
struct package_conf_item {
std::shared_ptr<PackageConf> first_pkgconf;
std::shared_ptr<Task> first_pull_task = std::make_shared<Task>();
std::shared_ptr<Task> first_tarball_task = std::make_shared<Task>();
std::shared_ptr<GitCommit> packaging_commit = std::make_shared<GitCommit>();
std::shared_ptr<GitCommit> upstream_commit = std::make_shared<GitCommit>();
};
};
#endif // CI_LOGIC_H

339
cpp/common.cpp
View File

@ -1,339 +0,0 @@
// Copyright (C) 2024-2025 Simon Quigley <tsimonq2@ubuntu.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
#include "common.h"
#include "utilities.h"
#include "/usr/include/archive.h"
#include <archive_entry.h>
#include <iostream>
#include <fstream>
#include <sstream>
#include <stdexcept>
#include <cstdio>
#include <cstdlib>
#include <regex>
#include <chrono>
#include <ctime>
#include <mutex>
#include <unordered_set>
#include <QProcess>
// Define the global 'verbose' variable
bool verbose = false;
// Logger function implementations
void log_info(const std::string &msg) {
std::cout << "[INFO] " << msg << "\n";
}
void log_warning(const std::string &msg) {
std::cerr << "[WARNING] " << msg << "\n";
}
void log_error(const std::string &msg) {
std::cerr << "[ERROR] " << msg << "\n";
}
void log_verbose(const std::string &msg) {
if (verbose) {
std::cout << "[VERBOSE] " << msg << "\n";
}
}
namespace fs = std::filesystem;
bool run_command(const std::vector<std::string> &cmd,
const std::optional<std::filesystem::path> &cwd,
bool show_output,
std::shared_ptr<Log> log) {
if (cmd.empty()) {
throw std::runtime_error("Command is empty");
}
QProcess process;
// Set the working directory if provided
if (cwd) {
process.setWorkingDirectory(QString::fromStdString(cwd->string()));
}
// Set up the environment (if needed)
QProcessEnvironment env = QProcessEnvironment::systemEnvironment();
process.setProcessEnvironment(env);
// Extract executable and arguments
QString program = QString::fromStdString(cmd[0]);
QStringList arguments;
for (size_t i = 1; i < cmd.size(); ++i) {
arguments << QString::fromStdString(cmd[i]);
}
// Start the command
process.start(program, arguments);
if (!process.waitForStarted()) {
throw std::runtime_error("Failed to start the command: " + program.toStdString());
}
// Stream output while the process is running
while (process.state() == QProcess::Running) {
if (process.waitForReadyRead()) {
QByteArray output = process.readAllStandardOutput();
QByteArray error = process.readAllStandardError();
if (log) {
log->append(output.toStdString());
log->append(error.toStdString());
}
if (show_output) {
std::cout << output.toStdString();
std::cerr << error.toStdString();
}
}
}
// Wait for the process to finish
process.waitForFinished();
// Capture return code and errors
if (process.exitStatus() != QProcess::NormalExit || process.exitCode() != 0) {
QByteArray error_output = process.readAllStandardError();
std::string error_message = "Command failed with exit code: " + std::to_string(process.exitCode());
if (!error_output.isEmpty()) {
error_message += "\nError Output: " + error_output.toStdString();
}
throw std::runtime_error(error_message);
}
return true;
}
// Function to extract excluded files from a copyright file
std::vector<std::string> extract_files_excluded(const std::string& filepath) {
std::ifstream file(filepath);
if (!file.is_open()) {
throw std::runtime_error("Failed to open file: " + filepath);
}
std::vector<std::string> files_excluded;
std::string line;
std::regex files_excluded_pattern(R"(Files-Excluded:\s*(.*))");
bool in_files_excluded = false;
while (std::getline(file, line)) {
if (std::regex_match(line, files_excluded_pattern)) {
in_files_excluded = true;
std::smatch match;
if (std::regex_search(line, match, files_excluded_pattern) && match.size() > 1) {
files_excluded.emplace_back(match[1]);
}
} else if (in_files_excluded) {
if (!line.empty() && (line[0] == ' ' || line[0] == '\t')) {
files_excluded.emplace_back(line.substr(1));
} else {
break; // End of Files-Excluded block
}
}
}
return files_excluded;
}
// Function to create a tarball
void create_tarball(const std::string& tarballPath, const std::string& directory, const std::vector<std::string>& exclusions, std::shared_ptr<Log> log) {
log->append("Creating tarball: " + tarballPath);
struct archive* a = archive_write_new();
if (!a) {
throw std::runtime_error("Failed to create a new archive.");
}
if (archive_write_add_filter_gzip(a) != ARCHIVE_OK) {
std::string err = "Failed to add gzip filter: ";
err += archive_error_string(a);
archive_write_free(a);
throw std::runtime_error(err);
}
if (archive_write_set_format_pax_restricted(a) != ARCHIVE_OK) {
std::string err = "Failed to set format: ";
err += archive_error_string(a);
archive_write_free(a);
throw std::runtime_error(err);
}
if (archive_write_open_filename(a, tarballPath.c_str()) != ARCHIVE_OK) {
std::string err = "Could not open tarball for writing: ";
err += archive_error_string(a);
archive_write_free(a);
throw std::runtime_error(err);
}
// Initialize a set to track added relative paths to prevent duplication
std::unordered_set<std::string> added_paths;
// Iterate through the directory recursively without following symlinks
for (auto it = fs::recursive_directory_iterator(
directory,
fs::directory_options::skip_permission_denied);
it != fs::recursive_directory_iterator(); ++it) {
const auto& path = it->path();
std::error_code ec;
fs::path relative_path = fs::relative(path, directory, ec);
if (ec) {
log->append("Failed to compute relative path for: " + path.string() + " Error: " + ec.message());
continue;
}
// Normalize the relative path to avoid discrepancies
fs::path normalized_relative_path = relative_path.lexically_normal();
std::string relative_path_str = normalized_relative_path.string();
// Check if this path has already been added
if (!added_paths.insert(relative_path_str).second) {
log->append("Duplicate path detected and skipped: " + relative_path_str);
continue; // Skip adding this duplicate path
}
// Exclusion logic (if any exclusions are provided)
bool excluded = std::any_of(exclusions.begin(), exclusions.end(), [&relative_path_str](const std::string& exclusion) {
return relative_path_str.find(exclusion) != std::string::npos;
});
if (excluded) { continue; }
fs::file_status fstatus = it->symlink_status(ec);
if (ec) {
log->append("Failed to get file status for: " + path.string() + " Error: " + ec.message());
continue;
}
struct archive_entry* entry = archive_entry_new();
if (!entry) {
log->append("Failed to create archive entry for: " + path.string());
archive_write_free(a);
throw std::runtime_error("Failed to create archive entry.");
}
std::string entry_path = relative_path_str;
if (fs::is_directory(fstatus)) {
// Ensure the directory pathname ends with '/'
if (!entry_path.empty() && entry_path.back() != '/') {
entry_path += '/';
}
archive_entry_set_pathname(entry, entry_path.c_str());
} else {
archive_entry_set_pathname(entry, entry_path.c_str());
}
// Set file type, permissions, and size
if (fs::is_regular_file(fstatus)) {
// Regular file
uintmax_t filesize = fs::file_size(path, ec);
if (ec) {
log->append("Cannot get file size for: " + path.string() + " Error: " + ec.message());
archive_entry_free(entry);
continue;
}
archive_entry_set_size(entry, static_cast<off_t>(filesize));
archive_entry_set_filetype(entry, AE_IFREG);
archive_entry_set_perm(entry, static_cast<mode_t>(fstatus.permissions()));
}
else if (fs::is_symlink(fstatus)) {
fs::path target = fs::read_symlink(path, ec);
if (ec) {
log->append("Cannot read symlink for: " + path.string() + " Error: " + ec.message());
archive_entry_free(entry);
continue;
}
archive_entry_set_symlink(entry, target.c_str());
archive_entry_set_filetype(entry, AE_IFLNK);
archive_entry_set_perm(entry, static_cast<mode_t>(fstatus.permissions()));
}
else if (fs::is_directory(fstatus)) {
archive_entry_set_size(entry, 0);
archive_entry_set_filetype(entry, AE_IFDIR);
archive_entry_set_perm(entry, static_cast<mode_t>(fstatus.permissions()));
}
else {
log->append("Unsupported file type for: " + path.string());
archive_entry_free(entry);
continue;
}
// Retrieve and set the modification time
fs::file_time_type ftime = fs::last_write_time(path, ec);
std::time_t mtime;
if (ec) {
log->append("Failed to get last write time for: " + path.string() + " Error: " + ec.message());
// Obtain current UTC time as fallback
auto now = std::chrono::system_clock::now();
mtime = std::chrono::system_clock::to_time_t(now);
log->append("Setting default mtime (current UTC time) for: " + path.string());
} else {
mtime = to_time_t(ftime);
}
archive_entry_set_mtime(entry, mtime, 0);
if (archive_write_header(a, entry) != ARCHIVE_OK) {
log->append("Failed to write header for: " + path.string() + " Error: " + archive_error_string(a));
archive_entry_free(entry);
continue;
}
if (fs::is_regular_file(fstatus)) {
std::ifstream fileStream(path, std::ios::binary);
if (!fileStream) {
log->append("Failed to open file for reading: " + path.string());
archive_entry_free(entry);
continue;
}
const std::size_t bufferSize = 8192;
char buffer[bufferSize];
while (fileStream) {
fileStream.read(buffer, bufferSize);
std::streamsize bytesRead = fileStream.gcount();
if (bytesRead > 0) {
if (archive_write_data(a, buffer, static_cast<size_t>(bytesRead)) < 0) {
log->append("Failed to write data for: " + path.string() + " Error: " + archive_error_string(a));
break;
}
}
}
if (fileStream.bad()) {
log->append("Error reading file: " + path.string());
}
}
archive_entry_free(entry);
}
if (archive_write_close(a) != ARCHIVE_OK) {
std::string err = "Failed to close archive: ";
err += archive_error_string(a);
archive_write_free(a);
throw std::runtime_error(err);
}
if (archive_write_free(a) != ARCHIVE_OK) {
std::string err = "Failed to free archive: ";
err += archive_error_string(a);
throw std::runtime_error(err);
}
log->append("Tarball created and compressed: " + tarballPath);
}

89
cpp/common.h
View File

@ -1,89 +0,0 @@
// Copyright (C) 2024 Simon Quigley <tsimonq2@ubuntu.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
#ifndef COMMON_H
#define COMMON_H
#include "utilities.h"
#include <string>
#include <optional>
#include <filesystem>
#include <shared_mutex>
#include <mutex>
#include <vector>
#include <regex>
namespace fs = std::filesystem;
class Task;
class Log {
private:
std::string data = "";
mutable std::shared_mutex lock_;
std::weak_ptr<Task> task_context_;
std::string last_data_str = "";
public:
void append(const std::string& str) {
std::unique_lock lock(lock_);
std::string log_str = str.ends_with('\n') ? str : str + '\n';
if (str.empty() || last_data_str == log_str) { return; }
else if (str.contains("dpkg-source: warning: ignoring deletion of file")) { return; }
data += std::format("[{}] {}", get_current_utc_time("%Y-%m-%dT%H:%M:%SZ"), log_str);
last_data_str = log_str;
}
void set_log(const std::string& str) {
std::unique_lock lock(lock_);
data = str;
}
std::string get() const {
std::shared_lock lock(lock_);
return std::regex_replace(data, std::regex(R"(^\s+)"), "");
}
void assign_task_context(std::shared_ptr<Task> task) {
task_context_ = task;
}
std::shared_ptr<Task> get_task_context() const {
return task_context_.lock();
}
};
// Logger functions
extern bool verbose;
void log_info(const std::string &msg);
void log_warning(const std::string &msg);
void log_error(const std::string &msg);
void log_verbose(const std::string &msg);
// Function to run a command with optional working directory and show output
bool run_command(const std::vector<std::string> &cmd,
const std::optional<fs::path> &cwd = std::nullopt,
bool show_output = false,
std::shared_ptr<Log> log = nullptr);
// Function to extract excluded files from a copyright file
std::vector<std::string> extract_files_excluded(const std::string& filepath);
// Function to create a tarball
void create_tarball(const std::string& tarballPath,
const std::string& directory,
const std::vector<std::string>& exclusions,
std::shared_ptr<Log> log = nullptr);
#endif // COMMON_H

2
cpp/lintian-ppa.cpp
View File

@ -1,4 +1,4 @@
#include "common.h"
#include "utilities.h"
#include "ci_logic.h"
#include <yaml-cpp/yaml.h>
#include <filesystem>

82
cpp/lubuntuci_lib.cpp
View File

@ -1,82 +0,0 @@
// Copyright (C) 2024-2025 Simon Quigley <tsimonq2@ubuntu.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
#include "lubuntuci_lib.h"
#include "ci_logic.h"
#include "common.h"
#include <yaml-cpp/yaml.h>
#include <filesystem>
#include <iostream>
#include <vector>
#include <string>
#include <mutex>
#include <git2.h>
namespace fs = std::filesystem;
/**
* list_known_repos():
* Make sure we call CiLogic::init_global() before reading
* the config, otherwise the config node will be empty.
*/
std::vector<std::shared_ptr<PackageConf>> LubuntuCI::list_known_repos(int page, int per_page, const std::string& sort_by, const std::string& sort_order)
{
cilogic.init_global();
if (page == 0 || per_page == 0 || sort_by.empty() || sort_order.empty()) { return cilogic.get_config(); }
return cilogic.get_config("", page, per_page, sort_by, sort_order);
}
/**
* pull_repo():
* - We do not call init_global() here because list_known_repos()
* or build_repo() might do it. But calling it again is safe.
*/
bool LubuntuCI::pull_repo(const std::string &repo_name, std::shared_ptr<Log> log)
{
log->append("Ensuring the global config is initialized...\n");
cilogic.init_global();
log->append("Global config is initialized. Getting the configs for the package name...\n");
auto pkgconfs = cilogic.get_config(repo_name);
log->append("Configs retrieved. Performing the pull...\n");
return cilogic.pull_project(pkgconfs.at(0), log);
}
/**
* create_project_tarball
*/
bool LubuntuCI::create_project_tarball(const std::string &repo_name, std::shared_ptr<Log> log)
{
cilogic.init_global();
log->append("Global config is initialized. Getting the configs for the package name...\n");
auto pkgconfs = cilogic.get_config(repo_name);
log->append("Configs retrieved. Performing the tarball creation...\n");
return cilogic.create_project_tarball(pkgconfs.at(0), log);
}
/**
* build_repo():
* - Also safely calls init_global().
* - Reads skip_dput from config if present (default = false).
*/
bool LubuntuCI::build_repo(const std::string &repo_name, std::shared_ptr<Log> log)
{
cilogic.init_global();
bool success = true;
for (auto pkgconf : cilogic.get_config(repo_name)) {
const auto [build_success, changes_files] = cilogic.build_project(pkgconf, log);
success = success && build_success && cilogic.upload_and_lint(pkgconf, changes_files, false);
}
return success;
}

48
cpp/lubuntuci_lib.h
View File

@ -1,48 +0,0 @@
// Copyright (C) 2024-2025 Simon Quigley <tsimonq2@ubuntu.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
#ifndef LUBUNTUCI_LIB_H
#define LUBUNTUCI_LIB_H
#include <string>
#include <vector>
#include "ci_logic.h"
class LubuntuCI {
public:
/**
* List all known repositories from the merged config.
*/
std::vector<std::shared_ptr<PackageConf>> list_known_repos(int page = 0,
int per_page = 0,
const std::string& sort_by = "",
const std::string& sort_order = "");
/**
* Pull a specific repository by name (returns true on success).
*/
bool pull_repo(const std::string &repo_name, std::shared_ptr<Log> log = NULL);
bool create_project_tarball(const std::string &repo_name, std::shared_ptr<Log> log);
/**
* Build a specific repository by name (returns true on success).
*/
bool build_repo(const std::string &repo_name, std::shared_ptr<Log> log = NULL);
CiLogic cilogic = CiLogic();
};
#endif // LUBUNTUCI_LIB_H

123
cpp/task_queue.cpp
View File

@ -33,22 +33,16 @@ void TaskQueue::enqueue(std::shared_ptr<JobStatus> jobstatus,
auto now = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::system_clock::now().time_since_epoch())
.count();
// Create the task
std::shared_ptr<Task> task_ptr = std::make_shared<Task>(jobstatus, now, packageconf);
task_ptr->func = [task_func, self_weak = std::weak_ptr<Task>(task_ptr)](std::shared_ptr<Log> log) {
std::shared_ptr<Task> task_locked = self_weak.lock();
if (task_locked) {
log->assign_task_context(task_locked);
task_ptr->func = [task_func, self_weak = std::weak_ptr<Task>(task_ptr)](std::shared_ptr<Log> log) mutable {
if (auto task_locked = self_weak.lock())
task_func(log);
}
};
packageconf->assign_task(jobstatus, task_ptr, packageconf);
std::unique_lock<std::mutex> lock(tasks_mutex_);
tasks_.emplace(task_ptr);
}
cv_.notify_all(); // Notify worker threads
cv_.notify_all();
}
void TaskQueue::start() {
@ -59,14 +53,12 @@ void TaskQueue::start() {
}
void TaskQueue::stop() {
{
std::unique_lock<std::mutex> tasks_lock(tasks_mutex_);
std::unique_lock<std::mutex> packages_lock(running_packages_mutex_);
std::unique_lock<std::mutex> running_tasks_lock(running_tasks_mutex_);
{
std::unique_lock<std::mutex> lock(tasks_mutex_);
stop_ = true;
}
cv_.notify_all(); // Wake up all threads
for (auto& worker : workers_) {
cv_.notify_all();
for (auto &worker : workers_) {
if (worker.joinable()) {
worker.join();
}
@ -88,63 +80,42 @@ void TaskQueue::worker_thread() {
while (true) {
std::shared_ptr<Task> task_to_execute;
{
std::lock_guard<std::mutex> tasks_lock(tasks_mutex_);
std::unique_lock<std::mutex> lock(tasks_mutex_);
cv_.wait(lock, [this] { return stop_ || !tasks_.empty(); });
if (stop_ && tasks_.empty()) return;
auto it = tasks_.begin();
bool found_valid = false;
// Iterate through the set until a valid task is found
while (it != tasks_.end()) {
int package_id = (*it)->get_parent_packageconf()->package->id;
{
std::shared_ptr<Task> it_task = *it;
task_to_execute = it_task;
}
int package_id = task_to_execute->get_parent_packageconf()->package->id;
{
std::lock_guard<std::mutex> lock(running_packages_mutex_);
auto running_package_it = std::find_if(running_packages_.begin(), running_packages_.end(),
[&package_id](const std::shared_ptr<Package>& package) { return package->id == package_id; });
if (running_package_it != running_packages_.end()) {
++it; // Move to the next task
std::lock_guard<std::mutex> pkg_lock(running_packages_mutex_);
auto running_it = std::find_if(running_packages_.begin(), running_packages_.end(),
[package_id](const std::shared_ptr<Package> &pkg) { return pkg->id == package_id; });
if (running_it != running_packages_.end()) {
++it;
continue;
}
}
// Task is valid to execute
found_valid = true;
it = tasks_.erase(it);
task_to_execute = *it;
tasks_.erase(it);
break;
}
if (!found_valid) { continue; }
}
if (!task_to_execute || !task_to_execute->func) continue;
else {
{
std::lock_guard<std::mutex> packages_lock(running_packages_mutex_);
running_packages_.insert(task_to_execute->get_parent_packageconf()->package);
}
{
std::lock_guard<std::mutex> tasks_lock(running_tasks_mutex_);
running_tasks_.insert(task_to_execute);
}
}
// Set the start time
{
auto now = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::system_clock::now().time_since_epoch())
.count();
task_to_execute->start_time = now;
std::lock_guard<std::mutex> pkg_lock(running_packages_mutex_);
running_packages_.insert(task_to_execute->get_parent_packageconf()->package);
}
{
std::lock_guard<std::mutex> rt_lock(running_tasks_mutex_);
running_tasks_.insert(task_to_execute);
}
auto now = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
task_to_execute->start_time = now;
try {
task_to_execute->func(task_to_execute->log); // Execute the task
task_to_execute->func(task_to_execute->log);
task_to_execute->successful = true;
} catch (const std::exception& e) {
} catch (const std::exception &e) {
task_to_execute->successful = false;
std::ostringstream oss;
oss << "Exception type: " << typeid(e).name() << "\n"
@ -154,39 +125,27 @@ void TaskQueue::worker_thread() {
task_to_execute->successful = false;
task_to_execute->log->append("Unknown exception occurred");
}
now = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
task_to_execute->finish_time = now;
{
auto now = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::system_clock::now().time_since_epoch())
.count();
task_to_execute->finish_time = now;
}
{
// Remove the task from running_tasks_
std::lock_guard<std::mutex> lock(running_tasks_mutex_);
std::lock_guard<std::mutex> rt_lock(running_tasks_mutex_);
int id = task_to_execute->id;
auto running_task_it = std::find_if(running_tasks_.begin(), running_tasks_.end(),
[&id](const std::shared_ptr<Task>& task) { return task->id == id; });
if (running_task_it != running_tasks_.end()) {
running_tasks_.erase(running_task_it);
auto it = std::find_if(running_tasks_.begin(), running_tasks_.end(),
[id](const std::shared_ptr<Task> &task) { return task->id == id; });
if (it != running_tasks_.end()) {
running_tasks_.erase(it);
}
}
{
// Remove packageconf from running_packages_ by id
std::lock_guard<std::mutex> lock(running_packages_mutex_);
std::lock_guard<std::mutex> pkg_lock(running_packages_mutex_);
int package_id = task_to_execute->get_parent_packageconf()->package->id;
auto running_package_it = std::find_if(running_packages_.begin(), running_packages_.end(),
[&package_id](const std::shared_ptr<Package>& package) { return package->id == package_id; });
if (running_package_it != running_packages_.end()) {
running_packages_.erase(running_package_it);
auto it = std::find_if(running_packages_.begin(), running_packages_.end(),
[package_id](const std::shared_ptr<Package> &pkg) { return pkg->id == package_id; });
if (it != running_packages_.end()) {
running_packages_.erase(it);
}
}
// Save to the database at the end
task_to_execute->save(0);
}
}

6
cpp/update-maintainer.cpp
View File

@ -1,6 +1,4 @@
// cpp/update-maintainer.cpp
// Copyright (C) 2024 Simon Quigley <tsimonq2@ubuntu.com>
// Copyright (C) 2024-2025 Simon Quigley <tsimonq2@ubuntu.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -15,7 +13,7 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
#include "lubuntuci_lib.h"
#include "ci_logic.h"
#include <iostream>
int main(int argc, char** argv) {

315
cpp/utilities.cpp
View File

@ -14,8 +14,9 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
#include "utilities.h"
#include "common.h"
#include "/usr/include/archive.h"
#include <archive_entry.h>
#include <fstream>
#include <iostream>
#include <filesystem>
@ -25,17 +26,16 @@
#include <sys/stat.h>
#include <sstream>
#include <random>
#include <queue>
#include <ranges>
#include <format> // for std::format in C++20/23
#include <format>
#include <unordered_set>
namespace fs = std::filesystem;
bool verbose = false;
// Define a semaphore with a maximum of 10 concurrent jobs
static std::counting_semaphore<10> sem(10);
// Job queue and synchronization primitives
static std::queue<std::function<void()>> job_queue;
static std::mutex queue_mutex;
static std::atomic<bool> daemon_running{false};
@ -296,3 +296,308 @@ void run_task_every(std::stop_token _stop_token, int interval_minutes, std::func
std::this_thread::sleep_for(std::chrono::minutes(interval_minutes));
}
}
// Logger function implementations
void log_info(const std::string &msg) {
std::cout << "[INFO] " << msg << "\n";
}
void log_warning(const std::string &msg) {
std::cerr << "[WARNING] " << msg << "\n";
}
void log_error(const std::string &msg) {
std::cerr << "[ERROR] " << msg << "\n";
}
void log_verbose(const std::string &msg) {
if (verbose) {
std::cout << "[VERBOSE] " << msg << "\n";
}
}
namespace fs = std::filesystem;
bool run_command(const std::vector<std::string> &cmd,
const std::optional<std::filesystem::path> &cwd,
bool show_output,
std::shared_ptr<Log> log) {
if (cmd.empty()) {
throw std::runtime_error("Command is empty");
}
QProcess process;
// Set the working directory if provided
if (cwd) {
process.setWorkingDirectory(QString::fromStdString(cwd->string()));
}
// Set up the environment (if needed)
QProcessEnvironment env = QProcessEnvironment::systemEnvironment();
process.setProcessEnvironment(env);
// Extract executable and arguments
QString program = QString::fromStdString(cmd[0]);
QStringList arguments;
for (size_t i = 1; i < cmd.size(); ++i) {
arguments << QString::fromStdString(cmd[i]);
}
// Start the command
process.start(program, arguments);
if (!process.waitForStarted()) {
throw std::runtime_error("Failed to start the command: " + program.toStdString());
}
// Stream output while the process is running
while (process.state() == QProcess::Running) {
if (process.waitForReadyRead()) {
QByteArray output = process.readAllStandardOutput();
QByteArray error = process.readAllStandardError();
if (log) {
log->append(output.toStdString());
log->append(error.toStdString());
}
if (show_output) {
std::cout << output.toStdString();
std::cerr << error.toStdString();
}
}
}
// Wait for the process to finish
process.waitForFinished();
// Capture return code and errors
if (process.exitStatus() != QProcess::NormalExit || process.exitCode() != 0) {
QByteArray error_output = process.readAllStandardError();
std::string error_message = "Command failed with exit code: " + std::to_string(process.exitCode());
if (!error_output.isEmpty()) {
error_message += "\nError Output: " + error_output.toStdString();
}
throw std::runtime_error(error_message);
}
return true;
}
// Function to extract excluded files from a copyright file
std::vector<std::string> extract_files_excluded(const std::string& filepath) {
std::ifstream file(filepath);
if (!file.is_open()) {
throw std::runtime_error("Failed to open file: " + filepath);
}
std::vector<std::string> files_excluded;
std::string line;
std::regex files_excluded_pattern(R"(Files-Excluded:\s*(.*))");
bool in_files_excluded = false;
while (std::getline(file, line)) {
if (std::regex_match(line, files_excluded_pattern)) {
in_files_excluded = true;
std::smatch match;
if (std::regex_search(line, match, files_excluded_pattern) && match.size() > 1) {
files_excluded.emplace_back(match[1]);
}
} else if (in_files_excluded) {
if (!line.empty() && (line[0] == ' ' || line[0] == '\t')) {
files_excluded.emplace_back(line.substr(1));
} else {
break; // End of Files-Excluded block
}
}
}
return files_excluded;
}
// Function to create a tarball
void create_tarball(const std::string& tarballPath, const std::string& directory, const std::vector<std::string>& exclusions, std::shared_ptr<Log> log) {
log->append("Creating tarball: " + tarballPath);
struct archive* a = archive_write_new();
if (!a) {
throw std::runtime_error("Failed to create a new archive.");
}
if (archive_write_add_filter_gzip(a) != ARCHIVE_OK) {
std::string err = "Failed to add gzip filter: ";
err += archive_error_string(a);
archive_write_free(a);
throw std::runtime_error(err);
}
if (archive_write_set_format_pax_restricted(a) != ARCHIVE_OK) {
std::string err = "Failed to set format: ";
err += archive_error_string(a);
archive_write_free(a);
throw std::runtime_error(err);
}
if (archive_write_open_filename(a, tarballPath.c_str()) != ARCHIVE_OK) {
std::string err = "Could not open tarball for writing: ";
err += archive_error_string(a);
archive_write_free(a);
throw std::runtime_error(err);
}
// Initialize a set to track added relative paths to prevent duplication
std::unordered_set<std::string> added_paths;
// Iterate through the directory recursively without following symlinks
for (auto it = fs::recursive_directory_iterator(
directory,
fs::directory_options::skip_permission_denied);
it != fs::recursive_directory_iterator(); ++it) {
const auto& path = it->path();
std::error_code ec;
fs::path relative_path = fs::relative(path, directory, ec);
if (ec) {
log->append("Failed to compute relative path for: " + path.string() + " Error: " + ec.message());
continue;
}
// Normalize the relative path to avoid discrepancies
fs::path normalized_relative_path = relative_path.lexically_normal();
std::string relative_path_str = normalized_relative_path.string();
// Check if this path has already been added
if (!added_paths.insert(relative_path_str).second) {
log->append("Duplicate path detected and skipped: " + relative_path_str);
continue; // Skip adding this duplicate path
}
// Exclusion logic (if any exclusions are provided)
bool excluded = std::any_of(exclusions.begin(), exclusions.end(), [&relative_path_str](const std::string& exclusion) {
return relative_path_str.find(exclusion) != std::string::npos;
});
if (excluded) { continue; }
fs::file_status fstatus = it->symlink_status(ec);
if (ec) {
log->append("Failed to get file status for: " + path.string() + " Error: " + ec.message());
continue;
}
struct archive_entry* entry = archive_entry_new();
if (!entry) {
log->append("Failed to create archive entry for: " + path.string());
archive_write_free(a);
throw std::runtime_error("Failed to create archive entry.");
}
std::string entry_path = relative_path_str;
if (fs::is_directory(fstatus)) {
// Ensure the directory pathname ends with '/'
if (!entry_path.empty() && entry_path.back() != '/') {
entry_path += '/';
}
archive_entry_set_pathname(entry, entry_path.c_str());
} else {
archive_entry_set_pathname(entry, entry_path.c_str());
}
// Set file type, permissions, and size
if (fs::is_regular_file(fstatus)) {
// Regular file
uintmax_t filesize = fs::file_size(path, ec);
if (ec) {
log->append("Cannot get file size for: " + path.string() + " Error: " + ec.message());
archive_entry_free(entry);
continue;
}
archive_entry_set_size(entry, static_cast<off_t>(filesize));
archive_entry_set_filetype(entry, AE_IFREG);
archive_entry_set_perm(entry, static_cast<mode_t>(fstatus.permissions()));
}
else if (fs::is_symlink(fstatus)) {
fs::path target = fs::read_symlink(path, ec);
if (ec) {
log->append("Cannot read symlink for: " + path.string() + " Error: " + ec.message());
archive_entry_free(entry);
continue;
}
archive_entry_set_symlink(entry, target.c_str());
archive_entry_set_filetype(entry, AE_IFLNK);
archive_entry_set_perm(entry, static_cast<mode_t>(fstatus.permissions()));
}
else if (fs::is_directory(fstatus)) {
archive_entry_set_size(entry, 0);
archive_entry_set_filetype(entry, AE_IFDIR);
archive_entry_set_perm(entry, static_cast<mode_t>(fstatus.permissions()));
}
else {
log->append("Unsupported file type for: " + path.string());
archive_entry_free(entry);
continue;
}
// Retrieve and set the modification time
fs::file_time_type ftime = fs::last_write_time(path, ec);
std::time_t mtime;
if (ec) {
log->append("Failed to get last write time for: " + path.string() + " Error: " + ec.message());
// Obtain current UTC time as fallback
auto now = std::chrono::system_clock::now();
mtime = std::chrono::system_clock::to_time_t(now);
log->append("Setting default mtime (current UTC time) for: " + path.string());
} else {
mtime = to_time_t(ftime);
}
archive_entry_set_mtime(entry, mtime, 0);
if (archive_write_header(a, entry) != ARCHIVE_OK) {
log->append("Failed to write header for: " + path.string() + " Error: " + archive_error_string(a));
archive_entry_free(entry);
continue;
}
if (fs::is_regular_file(fstatus)) {
std::ifstream fileStream(path, std::ios::binary);
if (!fileStream) {
log->append("Failed to open file for reading: " + path.string());
archive_entry_free(entry);
continue;
}
const std::size_t bufferSize = 8192;
char buffer[bufferSize];
while (fileStream) {
fileStream.read(buffer, bufferSize);
std::streamsize bytesRead = fileStream.gcount();
if (bytesRead > 0) {
if (archive_write_data(a, buffer, static_cast<size_t>(bytesRead)) < 0) {
log->append("Failed to write data for: " + path.string() + " Error: " + archive_error_string(a));
break;
}
}
}
if (fileStream.bad()) {
log->append("Error reading file: " + path.string());
}
}
archive_entry_free(entry);
}
if (archive_write_close(a) != ARCHIVE_OK) {
std::string err = "Failed to close archive: ";
err += archive_error_string(a);
archive_write_free(a);
throw std::runtime_error(err);
}
if (archive_write_free(a) != ARCHIVE_OK) {
std::string err = "Failed to free archive: ";
err += archive_error_string(a);
throw std::runtime_error(err);
}
log->append("Tarball created and compressed: " + tarballPath);
}

74
cpp/utilities.h
View File

@ -18,11 +18,57 @@
#include <string>
#include <filesystem>
#include <mutex>
#include <regex>
#include <future>
#include <shared_mutex>
#include <semaphore>
#include <functional>
#include <git2.h>
#include <QProcess>
namespace fs = std::filesystem;
class Task;
// Time utilities
std::string get_current_utc_time(const std::string& format);
std::time_t to_time_t(const std::filesystem::file_time_type& ftime);
class Log {
private:
std::string data = "";
mutable std::shared_mutex lock_;
std::weak_ptr<Task> task_context_;
std::string last_data_str = "";
public:
void append(const std::string& str) {
std::unique_lock lock(lock_);
std::string log_str = str.ends_with('\n') ? str : str + '\n';
if (str.empty() || last_data_str == log_str) { return; }
else if (str.contains("dpkg-source: warning: ignoring deletion of file")) { return; }
data += std::format("[{}] {}", get_current_utc_time("%Y-%m-%dT%H:%M:%SZ"), log_str);
last_data_str = log_str;
}
void set_log(const std::string& str) {
std::unique_lock lock(lock_);
data = str;
}
std::string get() const {
std::shared_lock lock(lock_);
return std::regex_replace(data, std::regex(R"(^\s+)"), "");
}
void assign_task_context(std::shared_ptr<Task> task) {
task_context_ = task;
}
std::shared_ptr<Task> get_task_context() const {
return task_context_.lock();
}
};
// Function to read the entire content of a file into a string
std::string read_file(const std::filesystem::path& filePath);
@ -49,10 +95,6 @@ std::filesystem::path create_temp_directory();
// Function to copy a directory recursively
void copy_directory(const std::filesystem::path& source, const std::filesystem::path& destination);
// Time utilities
std::string get_current_utc_time(const std::string& format);
std::time_t to_time_t(const std::filesystem::file_time_type& ftime);
// String utilities
std::vector<std::string> split_string(const std::string& input, const std::string& delimiter);
std::string remove_suffix(const std::string& input, const std::string& suffix);
@ -65,3 +107,25 @@ std::pair<int, bool> get_version_from_codename(const std::string& codename);
void ensure_git_inited();
void run_task_every(std::stop_token _stop_token, int interval_minutes, std::function<void()> task);
// Logger functions
extern bool verbose;
void log_info(const std::string &msg);
void log_warning(const std::string &msg);
void log_error(const std::string &msg);
void log_verbose(const std::string &msg);
// Function to run a command with optional working directory and show output
bool run_command(const std::vector<std::string> &cmd,
const std::optional<fs::path> &cwd = std::nullopt,
bool show_output = false,
std::shared_ptr<Log> log = nullptr);
// Function to extract excluded files from a copyright file
std::vector<std::string> extract_files_excluded(const std::string& filepath);
// Function to create a tarball
void create_tarball(const std::string& tarballPath,
const std::string& directory,
const std::vector<std::string>& exclusions,
std::shared_ptr<Log> log = nullptr);

121
cpp/web_server.cpp
View File

@ -18,6 +18,8 @@
#include "sources_parser.h"
#include "naive_bayes_classifier.h"
#include "db_common.h"
#include "template_renderer.h"
#include "ci_logic.h"
// Qt includes
#include <QtHttpServer/QHttpServer>
@ -57,10 +59,6 @@
#include "build.h"
#include "binary_package_publishing_history.h"
// Local includes
#include "lubuntuci_lib.h"
#include "template_renderer.h"
constexpr QHttpServerResponder::StatusCode StatusCodeFound = QHttpServerResponder::StatusCode::Found;
static std::string timestamp_now()
@ -131,7 +129,8 @@ WebServer::WebServer(QObject *parent) : QObject(parent) {}
QHttpServerResponse bad_response(StatusCodeFound);
QHttpHeaders bad_response_headers;
bad_response_headers.replaceOrAppend(QHttpHeaders::WellKnownHeader::Location, "/unauthorized?base_url=" + base_url + "&redirect_to=" + current_path);
bad_response_headers.replaceOrAppend(QHttpHeaders::WellKnownHeader::Location,
"/unauthorized?base_url=" + base_url + "&redirect_to=" + current_path);
bad_response.setHeaders(bad_response_headers);
return bad_response;
@ -174,14 +173,14 @@ bool WebServer::start_server(quint16 port) {
}
archive proposed = proposed_opt.value();
std::shared_ptr<PackageConf> _tmp_pkg_conf = std::make_shared<PackageConf>();
std::shared_ptr<LubuntuCI> lubuntuci = std::make_shared<LubuntuCI>();
std::vector<std::shared_ptr<PackageConf>> all_repos = lubuntuci->list_known_repos();
// Use our new list_known_repos() method from CiLogic
std::shared_ptr<CiLogic> cilogic = std::make_shared<CiLogic>();
std::vector<std::shared_ptr<PackageConf>> all_repos = cilogic->list_known_repos();
task_queue = std::make_unique<TaskQueue>(6);
std::shared_ptr<std::map<std::string, std::shared_ptr<JobStatus>>> job_statuses = lubuntuci->cilogic.get_job_statuses();
std::shared_ptr<std::map<std::string, std::shared_ptr<JobStatus>>> job_statuses = cilogic->get_job_statuses();
task_queue->start();
// Load initial tokens
// Load initial tokens from the database
{
QSqlQuery load_tokens(get_thread_connection());
load_tokens.prepare("SELECT person.id, person.username, person.logo_url, person_token.token, person_token.expiry_date FROM person INNER JOIN person_token ON person.id = person_token.person_id");
@ -199,12 +198,12 @@ bool WebServer::start_server(quint16 port) {
}
}
expire_tokens_thread_ = std::jthread(run_task_every, 60, [this, lubuntuci] {
expire_tokens_thread_ = std::jthread(run_task_every, 60, [this, cilogic] {
QSqlQuery expired_tokens(get_thread_connection());
QString current_time = QDateTime::currentDateTime().toString(Qt::ISODate);
expired_tokens.prepare("DELETE FROM person_token WHERE expiry_date < :current_time");
expired_tokens.bindValue(":current_time", QDateTime::currentDateTime().toString(Qt::ISODate));
expired_tokens.bindValue(":current_time", current_time);
ci_query_exec(&expired_tokens);
for (auto it = _active_tokens.begin(); it != _active_tokens.end();) {
if (it.value() <= QDateTime::currentDateTime()) it = _active_tokens.erase(it);
@ -216,10 +215,9 @@ bool WebServer::start_server(quint16 port) {
}
});
process_sources_thread_ = std::jthread(run_task_every, 10, [this, all_repos, proposed, lubuntuci, job_statuses] {
process_sources_thread_ = std::jthread(run_task_every, 10, [this, all_repos, proposed, cilogic, job_statuses] {
for (auto pkgconf : all_repos) {
if (!pkgconf->can_check_source_upload()) { continue; }
if (!pkgconf->can_check_source_upload()) continue;
task_queue->enqueue(
job_statuses->at("source_check"),
[this, pkgconf, proposed](std::shared_ptr<Log> log) mutable {
@ -229,22 +227,17 @@ bool WebServer::start_server(quint16 port) {
found_in_ppa = true;
break;
}
if (!found_in_ppa) {
throw std::runtime_error("Not found in the PPA.");
}
if (!found_in_ppa) throw std::runtime_error("Not found in the PPA.");
},
pkgconf
);
pkgconf->sync();
}
});
process_binaries_thread_ = std::jthread(run_task_every, 15, [this, all_repos, proposed, lubuntuci, job_statuses] {
process_binaries_thread_ = std::jthread(run_task_every, 15, [this, all_repos, proposed, cilogic, job_statuses] {
for (auto pkgconf : all_repos) {
if (!pkgconf->can_check_builds()) { continue; }
if (!pkgconf->can_check_builds()) continue;
task_queue->enqueue(
job_statuses->at("build_check"),
[this, pkgconf, proposed](std::shared_ptr<Log> log) mutable {
@ -279,7 +272,7 @@ bool WebServer::start_server(quint16 port) {
////////////////////////////////////////////////////////////////
// /unauthorized?base_url=<base_url>&redirect_to=<redirect_to>
////////////////////////////////////////////////////////////////
http_server_.route("/unauthorized", [this, lubuntuci](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
http_server_.route("/unauthorized", [this, cilogic](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
// Extract data up front
auto query = req.query();
QString base_url = query.queryItemValue("base_url");
@ -335,7 +328,7 @@ bool WebServer::start_server(quint16 port) {
/////////////////
// /authcallback
/////////////////
http_server_.route("/authcallback", [this, lubuntuci](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
http_server_.route("/authcallback", [this, cilogic](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
// Extract data up front
auto query = req.query();
QString base_url = query.queryItemValue("base_url");
@ -476,7 +469,7 @@ bool WebServer::start_server(quint16 port) {
//////////////////////////////////////////
// Route "/"
//////////////////////////////////////////
http_server_.route("/", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
http_server_.route("/", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
{
QHttpServerResponse session_response = verify_session_token(req, req.headers());
if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); });
@ -492,13 +485,13 @@ bool WebServer::start_server(quint16 port) {
: query.queryItemValue("sort_order").toStdString();
return QtConcurrent::run([=, this]() {
auto all_repos = lubuntuci->list_known_repos();
auto all_repos = cilogic->list_known_repos();
int total_size = static_cast<int>(all_repos.size());
int total_pages = (per_page > 0)
? (total_size + per_page - 1) / per_page
: 1;
auto repos = lubuntuci->list_known_repos(page, per_page, sort_by, sort_order);
auto repos = cilogic->list_known_repos(page, per_page, sort_by, sort_order);
if (repos.empty() && total_size == 0) {
std::string err_html = R"(
<html>
@ -519,37 +512,31 @@ bool WebServer::start_server(quint16 port) {
{"total_pages", std::to_string(total_pages)}
};
std::map<std::string, std::vector<std::map<std::string, std::string>>> list_context;
std::vector<std::map<std::string, std::string>> reposVec;
for (const auto &r : repos) {
std::map<std::string, std::string> item;
std::string packaging_commit_str;
std::string upstream_commit_str;
if (r->packaging_commit) {
std::string commit_summary = r->packaging_commit->commit_summary;
if (commit_summary.size() > 40) {
if (commit_summary.size() > 40)
commit_summary = commit_summary.substr(0, 37) + "...";
}
packaging_commit_str = r->packaging_commit->commit_hash.substr(0, 7) +
std::format(" ({:%Y-%m-%d %H:%M:%S %Z})<br />", r->packaging_commit->commit_datetime) +
commit_summary;
}
if (r->upstream_commit) {
std::string commit_summary = r->upstream_commit->commit_summary;
if (commit_summary.size() > 40) {
if (commit_summary.size() > 40)
commit_summary = commit_summary.substr(0, 37) + "...";
}
upstream_commit_str = r->upstream_commit->commit_hash.substr(0, 7) +
std::format(" ({:%Y-%m-%d %H:%M:%S %Z})<br />", r->upstream_commit->commit_datetime) +
commit_summary;
}
std::string packaging_commit_url_str = (r->package ? r->package->packaging_browser : "") +
(r->packaging_commit ? r->packaging_commit->commit_hash : "");
std::string upstream_commit_url_str = (r->package ? r->package->upstream_browser : "") +
(r->upstream_commit ? r->upstream_commit->commit_hash : "");
item["id"] = std::to_string(r->id);
item["name"] = r->package->name;
item["branch_name"] = r->branch->name;
@ -597,7 +584,7 @@ bool WebServer::start_server(quint16 port) {
//////////////////////////////////////////
// /pull?repo=<id>
//////////////////////////////////////////
http_server_.route("/pull", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
http_server_.route("/pull", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
{
QHttpServerResponse session_response = verify_session_token(req, req.headers());
if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); });
@ -612,7 +599,7 @@ bool WebServer::start_server(quint16 port) {
}
int repo = std::stoi(repo_string.toStdString());
std::string msg = lubuntuci->cilogic.queue_pull_tarball({ lubuntuci->cilogic.get_packageconf_by_id(repo) }, task_queue, job_statuses);
std::string msg = cilogic->queue_pull_tarball({ cilogic->get_packageconf_by_id(repo) }, task_queue, job_statuses);
return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size()));
});
});
@ -620,7 +607,7 @@ bool WebServer::start_server(quint16 port) {
//////////////////////////////////////////
// /build?repo=<id>
//////////////////////////////////////////
http_server_.route("/build", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
http_server_.route("/build", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
{
QHttpServerResponse session_response = verify_session_token(req, req.headers());
if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); });
@ -635,7 +622,7 @@ bool WebServer::start_server(quint16 port) {
}
int repo = std::stoi(repo_string.toStdString());
std::string msg = lubuntuci->cilogic.queue_build_upload({ lubuntuci->cilogic.get_packageconf_by_id(repo) }, task_queue, job_statuses);
std::string msg = cilogic->queue_build_upload({ cilogic->get_packageconf_by_id(repo) }, task_queue, job_statuses);
return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size()));
});
});
@ -643,7 +630,7 @@ bool WebServer::start_server(quint16 port) {
//////////////////////////////////////////
// /pull-selected?repos=<ids>
//////////////////////////////////////////
http_server_.route("/pull-selected", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
http_server_.route("/pull-selected", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
{
QHttpServerResponse session_response = verify_session_token(req, req.headers());
if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); });
@ -667,7 +654,7 @@ bool WebServer::start_server(quint16 port) {
})
);
std::string msg = lubuntuci->cilogic.queue_pull_tarball(lubuntuci->cilogic.get_packageconfs_by_ids(repos), task_queue, job_statuses);
std::string msg = cilogic->queue_pull_tarball(cilogic->get_packageconfs_by_ids(repos), task_queue, job_statuses);
return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size()));
});
});
@ -675,7 +662,7 @@ bool WebServer::start_server(quint16 port) {
//////////////////////////////////////////
// /build-selected?repos=foo,bar,baz
//////////////////////////////////////////
http_server_.route("/build-selected", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
http_server_.route("/build-selected", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
{
QHttpServerResponse session_response = verify_session_token(req, req.headers());
if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); });
@ -699,7 +686,7 @@ bool WebServer::start_server(quint16 port) {
})
);
std::string msg = lubuntuci->cilogic.queue_build_upload(lubuntuci->cilogic.get_packageconfs_by_ids(repos), task_queue, job_statuses);
std::string msg = cilogic->queue_build_upload(cilogic->get_packageconfs_by_ids(repos), task_queue, job_statuses);
return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size()));
});
});
@ -707,7 +694,7 @@ bool WebServer::start_server(quint16 port) {
//////////////////////////////////////////
// /pull-and-build-selected?repos=foo,bar,baz
//////////////////////////////////////////
http_server_.route("/pull-and-build-selected", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
http_server_.route("/pull-and-build-selected", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
{
QHttpServerResponse session_response = verify_session_token(req, req.headers());
if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); });
@ -730,11 +717,11 @@ bool WebServer::start_server(quint16 port) {
return std::stoi(s);
})
);
auto pkgconfs = lubuntuci->cilogic.get_packageconfs_by_ids(repos);
auto pkgconfs = cilogic->get_packageconfs_by_ids(repos);
for (auto pkgconf : pkgconfs) pkgconf->clear_tasks();
std::string msg = lubuntuci->cilogic.queue_pull_tarball(pkgconfs, task_queue, job_statuses);
msg += lubuntuci->cilogic.queue_build_upload(pkgconfs, task_queue, job_statuses);
std::string msg = cilogic->queue_pull_tarball(pkgconfs, task_queue, job_statuses);
msg += cilogic->queue_build_upload(pkgconfs, task_queue, job_statuses);
return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size()));
});
});
@ -742,13 +729,13 @@ bool WebServer::start_server(quint16 port) {
//////////////////////////////////////////
// /pull-all
//////////////////////////////////////////
http_server_.route("/pull-all", [this, lubuntuci, all_repos, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
http_server_.route("/pull-all", [this, cilogic, all_repos, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
{
QHttpServerResponse session_response = verify_session_token(req, req.headers());
if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); });
}
return QtConcurrent::run([=, this]() {
std::string msg = lubuntuci->cilogic.queue_pull_tarball(all_repos, task_queue, job_statuses);
std::string msg = cilogic->queue_pull_tarball(all_repos, task_queue, job_statuses);
return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size()));
});
@ -757,13 +744,13 @@ bool WebServer::start_server(quint16 port) {
//////////////////////////////////////////
// /build-all
//////////////////////////////////////////
http_server_.route("/build-all", [this, lubuntuci, all_repos, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
http_server_.route("/build-all", [this, cilogic, all_repos, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
{
QHttpServerResponse session_response = verify_session_token(req, req.headers());
if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); });
}
return QtConcurrent::run([=, this]() {
std::string msg = lubuntuci->cilogic.queue_build_upload(all_repos, task_queue, job_statuses);
std::string msg = cilogic->queue_build_upload(all_repos, task_queue, job_statuses);
return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size()));
});
@ -772,15 +759,15 @@ bool WebServer::start_server(quint16 port) {
//////////////////////////////////////////
// /pull-and-build-all
//////////////////////////////////////////
http_server_.route("/pull-and-build-all", [this, lubuntuci, all_repos, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
http_server_.route("/pull-and-build-all", [this, cilogic, all_repos, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
{
QHttpServerResponse session_response = verify_session_token(req, req.headers());
if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); });
}
return QtConcurrent::run([=, this]() {
for (auto pkgconf : all_repos) pkgconf->clear_tasks();
std::string msg = lubuntuci->cilogic.queue_pull_tarball(all_repos, task_queue, job_statuses);
msg += lubuntuci->cilogic.queue_build_upload(all_repos, task_queue, job_statuses);
std::string msg = cilogic->queue_pull_tarball(all_repos, task_queue, job_statuses);
msg += cilogic->queue_build_upload(all_repos, task_queue, job_statuses);
return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size()));
});
@ -789,7 +776,7 @@ bool WebServer::start_server(quint16 port) {
//////////////////////////////////////////
// Serve static files from /static/<arg>
//////////////////////////////////////////
http_server_.route("/static/<arg>", [this, lubuntuci, job_statuses](const QString filename) -> QHttpServerResponse {
http_server_.route("/static/<arg>", [this, cilogic, job_statuses](const QString filename) -> QHttpServerResponse {
QString sanitized_filename = filename;
if (filename.contains("..") || filename.contains("../")) {
return QHttpServerResponse(QHttpServerResponder::StatusCode::BadRequest);
@ -827,7 +814,7 @@ bool WebServer::start_server(quint16 port) {
//////////////////////////////////////////
// /graph
//////////////////////////////////////////
http_server_.route("/graph", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
http_server_.route("/graph", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
{
QHttpServerResponse session_response = verify_session_token(req, req.headers());
if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); });
@ -875,7 +862,7 @@ bool WebServer::start_server(quint16 port) {
//////////////////////////////////////////
// /tasks
//////////////////////////////////////////
http_server_.route("/tasks", [this, lubuntuci, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
http_server_.route("/tasks", [this, cilogic, job_statuses](const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
{
QHttpServerResponse session_response = verify_session_token(req, req.headers());
if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); });
@ -911,7 +898,7 @@ bool WebServer::start_server(quint16 port) {
title_prefix = "Completed";
// gather tasks that have start_time > 0 and finish_time > 0
std::vector<std::shared_ptr<Task>> tasks_vector;
auto pkgconfs = lubuntuci->cilogic.get_packageconfs();
auto pkgconfs = cilogic->get_packageconfs();
for (auto &pkgconf : pkgconfs) {
for (auto &j : *job_statuses) {
if (!j.second) {
@ -967,7 +954,7 @@ bool WebServer::start_server(quint16 port) {
//////////////////////////////////////////
// /log/<TASK_ID>
//////////////////////////////////////////
http_server_.route("/log/<arg>", [this, lubuntuci, job_statuses](const QString _task_id, const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
http_server_.route("/log/<arg>", [this, cilogic, job_statuses](const QString _task_id, const QHttpServerRequest &req) -> QFuture<QHttpServerResponse> {
{
QHttpServerResponse session_response = verify_session_token(req, req.headers());
if (session_response.statusCode() == StatusCodeFound) return QtConcurrent::run([response = std::move(session_response)]() mutable { return std::move(response); });
@ -984,19 +971,12 @@ bool WebServer::start_server(quint16 port) {
std::string msg = "<html><body><h1>Invalid task ID specified.</h1></body></html>";
return QHttpServerResponse("text/html", QByteArray(msg.c_str(), (int)msg.size()));
}
std::string log_content = lubuntuci->cilogic.get_task_log(task_id);
std::map<std::string, std::vector<std::map<std::string, std::string>>> list_context;
std::string log_content = cilogic->get_task_log(task_id);
std::map<std::string, std::string> context;
std::map<std::string, std::vector<std::map<std::string, std::string>>> list_context;
context["title"] = "Task Logs";
context["log"] = log_content;
std::string final_html = TemplateRenderer::render_with_inheritance(
"log.html",
context,
list_context
);
std::string final_html = TemplateRenderer::render_with_inheritance("log.html", context, list_context);
return QHttpServerResponse("text/html", QByteArray(final_html.c_str(), (int)final_html.size()));
});
});
@ -1011,7 +991,6 @@ bool WebServer::start_server(quint16 port) {
key_file.open(QIODevice::ReadOnly);
ssl_config.setPrivateKey(QSslKey(&key_file, QSsl::Rsa, QSsl::Pem));
key_file.close();
ssl_config.setPeerVerifyMode(QSslSocket::VerifyNone);
ssl_config.setProtocol(QSsl::TlsV1_3);
ssl_server_.setSslConfiguration(ssl_config);