From 0393350b7043cf4ec489e56675361356d79250f8 Mon Sep 17 00:00:00 2001 From: Simon Quigley Date: Sat, 14 Dec 2024 23:11:06 -0600 Subject: [PATCH] Port build-packages to C++ --- build-packages | 441 ---------------- build-packages-cpp/CMakeLists.txt | 25 + build-packages-cpp/common.cpp | 116 ++++ build-packages-cpp/common.h | 10 + build-packages-cpp/main.cpp | 644 +++++++++++++++++++++++ build-packages-cpp/update_maintainer.cpp | 204 +++++++ build-packages-cpp/update_maintainer.h | 4 + 7 files changed, 1003 insertions(+), 441 deletions(-) delete mode 100755 build-packages create mode 100644 build-packages-cpp/CMakeLists.txt create mode 100644 build-packages-cpp/common.cpp create mode 100644 build-packages-cpp/common.h create mode 100644 build-packages-cpp/main.cpp create mode 100644 build-packages-cpp/update_maintainer.cpp create mode 100644 build-packages-cpp/update_maintainer.h diff --git a/build-packages b/build-packages deleted file mode 100755 index 88aa1f8..0000000 --- a/build-packages +++ /dev/null @@ -1,441 +0,0 @@ -#!/usr/bin/env python3 - -import subprocess -import io -import os -import sys -import yaml -import argparse -import logging -from datetime import datetime -import tarfile -import shutil -from git import Repo, GitCommandError -import tempfile -from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED -import fnmatch -import re -from debian.copyright import Header, Copyright -import uuid -from common import clean_old_logs - -BASE_DIR = "/srv/lubuntu-ci/repos" -DEBFULLNAME = "Lugito" -DEBEMAIL = "info@lubuntu.me" -OUTPUT_DIR = os.path.join(BASE_DIR, "build_output") -SUPPRESSED_LINTIAN_TAGS = [ - "orig-tarball-missing-upstream-signature", - "package-has-long-file-name", - "adopted-extended-field" -] -BASE_OUTPUT_DIR = "/srv/lubuntu-ci/output" -LOG_DIR = os.path.join(BASE_OUTPUT_DIR, "logs", "source_builds") -BASE_LINTIAN_DIR = os.path.join(BASE_OUTPUT_DIR, f".lintian.tmp.{str(uuid.uuid4())[:8]}") -REAL_LINTIAN_DIR = os.path.join(BASE_OUTPUT_DIR, "lintian") - -os.makedirs(LOG_DIR, exist_ok=True) -os.makedirs(OUTPUT_DIR, exist_ok=True) -os.makedirs(BASE_LINTIAN_DIR, exist_ok=True) - -current_time = datetime.utcnow().strftime("%Y%m%dT%H%M%S") -log_file = os.path.join(LOG_DIR, f"{current_time}.log") -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(levelname)s - %(message)s", - handlers=[ - logging.FileHandler(log_file), - logging.StreamHandler() - ] -) -logger = logging.getLogger("TimeBasedLogger") - -def run_command(cmd, cwd=None, env=None, show_output=False): - logging.info(f"Executing: {' '.join(cmd)} in {cwd or 'current directory'}") - try: - result = subprocess.run( - cmd, - cwd=cwd, - env=env, - check=True, - capture_output=True, - text=True - ) - if show_output: - if result.stdout: - logging.info(f"Output: {result.stdout.strip()}") - if result.stderr: - logging.warning(f"Output: {result.stderr.strip()}") - logging.info(f"Command succeeded: {' '.join(cmd)}") - except subprocess.CalledProcessError as e: - logging.error(f"Command failed: {' '.join(cmd)}") - logging.error(e.stderr) - logging.error(e.stdout) - raise - -def parse_version(changelog_path): - try: - epoch = None - - with open(changelog_path, "r") as f: - first_line = f.readline().strip() - - version_match = first_line.split("(")[1].split(")")[0] - upstream_version = version_match - - if ":" in version_match: - epoch, upstream_version = version_match.split(":", 1) - upstream_version = upstream_version.split("-")[0] - upstream_version = re.sub(r"(\+git[0-9]+)?(~[a-z]+)?$", "", upstream_version) - - # Log the results - current_date = datetime.now().strftime("%Y%m%d%H%M") - if epoch: - version = f"{epoch}:{upstream_version}+git{current_date}" - logging.info(f"Upstream version extracted: {epoch}:{upstream_version}") - logging.info(f"Going to use: {version}") - else: - version = f"{upstream_version}+git{current_date}" - logging.info(f"Upstream version extracted: {upstream_version}") - logging.info(f"Going to use: {version}") - - return version - except (IndexError, FileNotFoundError) as e: - logging.error(f"Error parsing version from {changelog_path}: {e}") - raise - -def get_exclusions(packaging): - exclusions = [] - with io.open(os.path.join(packaging, "debian/copyright"), "rt", encoding="utf-8") as f: - copyright_obj = Copyright(f) - for paragraph in copyright_obj.all_paragraphs(): - if isinstance(paragraph, Header): - if paragraph.files_excluded: - for file_name in paragraph.files_excluded: - exclusions.append(file_name) - break - return exclusions - -def create_tarball(name, source_dir, exclusions=[]): - tar_filename = f"{name}_MAIN.orig.tar.gz" - logging.info(f"Creating tarball: {tar_filename}") - exclusions.append(".git/") - - def exclusion_func(tarinfo): - for exclusion in exclusions: - if exclusion in tarinfo.name: - return None - - return tarinfo - - with tarfile.open(tar_filename, "w:gz") as tar: - tar.add(source_dir, arcname=os.path.basename(source_dir), filter=exclusion_func) - logging.info(f"Tarball created and compressed: {tar_filename}") - -def update_changelog(packaging_dir, release, version, env): - name = os.path.basename(packaging_dir) - logging.info(f"Updating changelog for {name} to version {version}-0ubuntu1~ppa1") - run_command(["git", "checkout", "debian/changelog"], cwd=packaging_dir) - cmd = [ - "dch", - "--distribution", release, - "--package", name, - "--newversion", f"{version}-0ubuntu1~ppa1", - "--urgency", "low", - "CI upload." - ] - run_command(cmd, cwd=packaging_dir, env=env) - -def build_package(packaging_dir, env, large): - name = os.path.basename(packaging_dir) - logging.info(f"Building source package for {name}") - - if large: - temp_dir = os.path.join(OUTPUT_DIR, f".tmp_{name}_{env['VERSION']}") - logging.warning(f"{name} is quite large and will not fit in /tmp, building at {temp_dir}") - if not os.path.exists(temp_dir): - os.mkdir(temp_dir) - else: - temp_dir = tempfile.mkdtemp() - - try: - temp_packaging_dir = os.path.join(temp_dir, name) - os.makedirs(temp_packaging_dir, exist_ok=True) - shutil.copytree(packaging_dir + "/debian", temp_packaging_dir + "/debian") - - tarball_name = f"{name}_{env['VERSION']}.orig.tar.gz" - tarball_source = os.path.join(BASE_DIR, tarball_name) - tarball_dest = os.path.join(temp_dir, tarball_name) - shutil.copyfile(tarball_source, tarball_dest) - - cmd_build = ["debuild", "--no-lintian", "-S", "-d", "-sa", "-nc"] - run_command(cmd_build, cwd=temp_packaging_dir, env=env) - run_command(["git", "checkout", "debian/changelog"], cwd=packaging_dir) - - pattern = f"{name}_{env['VERSION']}*" - for filename in os.listdir(temp_dir): - if fnmatch.fnmatch(filename, pattern): - source_file = os.path.join(temp_dir, filename) - dest_file = os.path.join(OUTPUT_DIR, filename) - shutil.copyfile(source_file, dest_file) - logging.info(f"Copied {filename} to {OUTPUT_DIR}") - - changes_files = [f for f in os.listdir(OUTPUT_DIR) if f.startswith(f"{name}_{env['VERSION']}") and f.endswith("_source.changes")] - finally: - shutil.rmtree(temp_dir) - - if changes_files: - changes_file = os.path.join(OUTPUT_DIR, changes_files[-1]) - logging.info(f"Built package, changes file: {changes_file}") - return changes_file - else: - logging.error("No changes file found after build.") - raise FileNotFoundError("Changes file not found.") - -def load_config(config_path): - try: - with open(config_path, "r") as f: - config = yaml.safe_load(f) - if "packages" not in config or "releases" not in config: - raise ValueError("Config file must contain 'packages' and 'releases' sections.") - return config - except Exception as e: - logging.error(f"Error loading config file: {e}") - sys.exit(1) - -def clone_or_update_repo(destination, repo_url, repo_branch=None): - if os.path.exists(destination): - logging.info(f"Repository already exists at {destination}, checking branch and remote URL.") - try: - repo = Repo(destination) - - current_remote_url = repo.remotes.origin.url - if current_remote_url != repo_url: - logging.info(f"Remote URL differs for {destination}. Removing and recloning.") - shutil.rmtree(destination) - else: - repo.git.reset("--hard", "HEAD") - current_branch = repo.active_branch.name - if repo_branch and current_branch != repo_branch: - logging.info(f"Branch differs for {destination}. Removing and recloning.") - shutil.rmtree(destination) - else: - logging.info(f"Repository matches desired remote and branch, pulling updates.") - repo.git.checkout(repo_branch or current_branch) - try: - repo.remotes.origin.pull() - repo.submodule_update(recursive=True) - logging.info(f"Pulled latest changes for {destination}") - except GitCommandError as e: - if 'non-fast-forward' in str(e): - logging.error(f"Pull failed due to non-fast-forward update: {e}") - logging.info(f"Removing repository {destination} and cloning again.") - shutil.rmtree(destination) - else: - logging.error(f"Pull failed for {destination}: {e}") - raise - else: - return - except Exception as e: - logging.error(f"Error updating repository {destination}: {e}") - logging.info(f"Removing repository {destination} and cloning again.") - shutil.rmtree(destination) - try: - logging.info(f"Cloning repository {repo_url} into {destination}") - repo = Repo.clone_from(repo_url, destination, recurse_submodules=True) - if repo_branch: - repo.git.checkout(repo_branch) - logging.info(f"Checked out {repo_branch} in {destination}") - except GitCommandError as e: - logging.error(f"Git clone failed for {repo_url}: {e}") - raise - -def publish_lintian(): - if os.path.exists(BASE_LINTIAN_DIR): - for root, dirs, files in os.walk(BASE_LINTIAN_DIR): - for file in files: - # Determine the source and destination paths - src_path = os.path.join(root, file) - rel_path = os.path.relpath(src_path, BASE_LINTIAN_DIR) - dest_path = os.path.join(REAL_LINTIAN_DIR, rel_path) - - # Ensure the destination directory exists - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - - # Copy the file - shutil.copy2(src_path, dest_path) - - # Remove the temporary directory - shutil.rmtree(BASE_LINTIAN_DIR) - -def run_source_lintian(name, sources_path): - logging.info(f"Running Lintian for {name}") - with tempfile.NamedTemporaryFile(mode='w+', suffix='.txt') as temp_file: - temp_file.write("\n".join(SUPPRESSED_LINTIAN_TAGS)) - temp_file.flush() - temp_file_path = temp_file.name - - cmd = [ - "lintian", - "-EvIL", - "+pedantic", - "--suppress-tags-from-file", - f"{temp_file_path}", - sources_path - ] - - result = subprocess.run( - cmd, - capture_output=True, - text=True - ) - - stderr, stdout = None, None - if result.stderr: - stderr = result.stderr.strip() - if result.stdout: - stdout = result.stdout.strip() - - lintian_output = None - if stderr == stdout: - lintian_output = stderr - else: - lintian_output = f"{stderr}\n{stdout}".strip() - - if lintian_output: - pkgdir = os.path.join(BASE_LINTIAN_DIR, name) - if not os.path.exists(pkgdir): - os.mkdir(pkgdir) - output_file = os.path.join(pkgdir, "source.txt") - with open(output_file, "a") as f: - f.write(lintian_output) - - logging.info(f"Lintian run for {name} is complete") - -def main(): - parser = argparse.ArgumentParser(description="Automate Lubuntu package builds.") - parser.add_argument("config", help="Path to the YAML configuration file.") - parser.add_argument("--skip-dput", action="store_true", help="Skip the dput upload step.") - parser.add_argument("--skip-cleanup", action="store_true", help="Skip removal of build_output.") - args = parser.parse_args() - - config = load_config(args.config) - packages = config["packages"] - releases = config["releases"] - - os.makedirs(BASE_DIR, exist_ok=True) - logging.info(f"Using base directory: {BASE_DIR}") - os.chdir(BASE_DIR) - - with ThreadPoolExecutor(max_workers=5) as executor: - def dput_source(name, upload_target, changes_files, devel_changes_files): - if changes_files: - hr_changes = ", ".join(changes_files) - logging.info(f"Uploading {hr_changes} to {upload_target} using dput") - cmd_upload = ["dput", upload_target] + changes_files - run_command(cmd_upload, cwd=OUTPUT_DIR) - logging.info(f"Completed upload of {hr_changes} to {upload_target}") - - for file in devel_changes_files: - if file: - futures.add(executor.submit(run_source_lintian, name, file)) - - def prepare_package(pkg): - name = pkg.get("name") - if not name: - logging.warning(f"Skipping package due to missing name: {pkg}") - return - upstream_url = pkg.get("upstream_url") or f"https://github.com/lxqt/{name}.git" - upstream_destination = os.path.join(BASE_DIR, f"upstream-{name}") - clone_or_update_repo(upstream_destination, upstream_url) - packaging_url = pkg.get("packaging_url") or f"https://git.lubuntu.me/Lubuntu/{name}-packaging.git" - packaging_branch = pkg.get("packaging_branch") or f"ubuntu/{releases[0]}" if releases else None - packaging_destination = os.path.join(BASE_DIR, name) - clone_or_update_repo(packaging_destination, packaging_url, packaging_branch) - exclusions = get_exclusions(packaging_destination) - create_tarball(name, upstream_destination, exclusions) - run_command(["update-maintainer"], cwd=packaging_destination) - futures.add(executor.submit(process_package, pkg)) - - def process_package(pkg): - name = pkg.get("name") - upload_target = pkg.get("upload_target", "ppa:lubuntu-ci/unstable-ci-proposed") - - if not name: - logging.warning(f"Skipping package due to missing name: {pkg}") - return [] - - package_changes = [] - - packaging_destination = os.path.join(BASE_DIR, name) - changelog_path = os.path.join(packaging_destination, "debian", "changelog") - version = parse_version(changelog_path) - - for release in releases: - logging.info(f"Building {name} for {release}") - try: - epoch, release_version = None, version - if ":" in version: - epoch, release_version = version.split(":", 1) - - release_version = f"{release_version}~{release}" - tarball_name = f"{name}_{release_version}.orig.tar.gz" - tarball_source = os.path.join(BASE_DIR, f"{name}_MAIN.orig.tar.gz") - tarball_dest = os.path.join(BASE_DIR, tarball_name) - shutil.copyfile(tarball_source, tarball_dest) - if epoch: - release_version = f"{epoch}:{release_version}" - - env = os.environ.copy() - env["DEBFULLNAME"] = DEBFULLNAME - env["DEBEMAIL"] = DEBEMAIL - env["VERSION"] = release_version - env["UPLOAD_TARGET"] = upload_target - - # Update changelog and build package - update_changelog(packaging_destination, release, release_version, env) - if epoch: - _, env["VERSION"] = release_version.split(":", 1) - changes_file = build_package(packaging_destination, env, pkg.get("large", False)) - if changes_file: - package_changes.append((changes_file, env)) - os.remove(os.path.join(BASE_DIR, tarball_name)) - - except Exception as e: - logging.error(f"Error processing package '{name}' for release '{release}': {e}") - - changes_files = [os.path.basename(cf) for cf, env in package_changes] - devel_changes_files = set(os.path.join(OUTPUT_DIR, file) if releases[0] in file else None for file in changes_files) - if args.skip_dput: - for changes_file in devel_changes_files: - if changes_file: - futures.add(executor.submit(run_source_lintian, name, changes_file)) - else: - if package_changes: - upload_target = package_changes[0][1]["UPLOAD_TARGET"] - futures.add(executor.submit(dput_source, name, upload_target, changes_files, devel_changes_files)) - - os.remove(os.path.join(BASE_DIR, f"{name}_MAIN.orig.tar.gz")) - - futures = set(executor.submit(prepare_package, pkg) for pkg in packages) - - while futures: - done, not_done = wait(futures, return_when=FIRST_COMPLETED) - - for future in done: - try: - result = future.result() - except Exception as e: - logging.exception("Task generated an exception") - finally: - futures.remove(future) - - if not args.skip_cleanup: - shutil.rmtree(OUTPUT_DIR) - logging.info("Publishing Lintian output...") - publish_lintian() - clean_old_logs(LOG_DIR) - - logging.info("Script completed successfully.") - -if __name__ == "__main__": - main() diff --git a/build-packages-cpp/CMakeLists.txt b/build-packages-cpp/CMakeLists.txt new file mode 100644 index 0000000..4ba80b1 --- /dev/null +++ b/build-packages-cpp/CMakeLists.txt @@ -0,0 +1,25 @@ +cmake_minimum_required(VERSION 3.21) +project(build-packages CXX) + +set(CMAKE_CXX_STANDARD 23) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +find_package(yaml-cpp REQUIRED) +find_package(PkgConfig REQUIRED) +pkg_check_modules(LIBGIT2 REQUIRED IMPORTED_TARGET libgit2) + +add_library(common SHARED common.cpp) +target_link_libraries(common yaml-cpp) + +add_library(update_maintainer SHARED update_maintainer.cpp) + +add_executable(build-packages main.cpp) +target_link_libraries(build-packages common update_maintainer PkgConfig::LIBGIT2 yaml-cpp) + +set(CMAKE_INSTALL_RPATH "$ORIGIN/lib") +set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) + +install(TARGETS common LIBRARY DESTINATION lib) +install(TARGETS update_maintainer LIBRARY DESTINATION lib) +install(TARGETS build-packages RUNTIME DESTINATION .) diff --git a/build-packages-cpp/common.cpp b/build-packages-cpp/common.cpp new file mode 100644 index 0000000..c40ba6d --- /dev/null +++ b/build-packages-cpp/common.cpp @@ -0,0 +1,116 @@ +#include "common.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static void log_info(const std::string &msg) { + std::cout << "[INFO] " << msg << "\n"; +} +static void log_error(const std::string &msg) { + std::cerr << "[ERROR] " << msg << "\n"; +} + +std::string parse_version(const fs::path &changelog_path) { + if (!fs::exists(changelog_path)) { + throw std::runtime_error("Changelog not found: " + changelog_path.string()); + } + std::ifstream f(changelog_path); + if (!f) throw std::runtime_error("Unable to open changelog"); + std::string first_line; + std::getline(f, first_line); + f.close(); + + size_t start = first_line.find('('); + size_t end = first_line.find(')'); + if (start == std::string::npos || end == std::string::npos) { + throw std::runtime_error("Invalid changelog format"); + } + std::string version_match = first_line.substr(start+1, end - (start+1)); + + std::string epoch; + std::string upstream_version = version_match; + if (auto pos = version_match.find(':'); pos != std::string::npos) { + epoch = version_match.substr(0, pos); + upstream_version = version_match.substr(pos+1); + } + if (auto pos = upstream_version.find('-'); pos != std::string::npos) { + upstream_version = upstream_version.substr(0, pos); + } + + std::regex git_regex("(\\+git[0-9]+)?(~[a-z]+)?$"); + upstream_version = std::regex_replace(upstream_version, git_regex, ""); + + auto t = std::time(nullptr); + std::tm tm = *std::gmtime(&t); + char buf[32]; + std::strftime(buf, sizeof(buf), "%Y%m%d%H%M", &tm); + std::string current_date = buf; + + std::string version; + if (!epoch.empty()) { + version = epoch + ":" + upstream_version + "+git" + current_date; + } else { + version = upstream_version + "+git" + current_date; + } + + return version; +} + +void run_command(const std::vector &cmd, const std::optional &cwd, bool show_output) { + std::string full_cmd; + for (const auto &c : cmd) { + full_cmd += c + " "; + } + if (cwd) { + full_cmd = "cd " + cwd->string() + " && " + full_cmd; + } + log_info("Executing: " + full_cmd); + int ret = std::system(full_cmd.c_str()); + if (ret != 0) { + log_error("Command failed: " + full_cmd); + throw std::runtime_error("Command failed"); + } + if (show_output) { + std::cout << "[INFO] Command succeeded: " + full_cmd << "\n"; + } +} + +void clean_old_logs(const fs::path &log_dir, int max_age_seconds) { + auto now = std::chrono::system_clock::now(); + for (auto &entry : fs::directory_iterator(log_dir)) { + if (fs::is_regular_file(entry)) { + auto ftime = fs::last_write_time(entry); + auto sctp = decltype(ftime)::clock::to_sys(ftime); + auto age = std::chrono::duration_cast(now - sctp).count(); + if (age > max_age_seconds) { + fs::remove(entry); + } + } + } +} + +void create_tarball(const std::string &name, const fs::path &source_dir, const std::vector &exclusions) { + std::string tar_filename = name + "_MAIN.orig.tar.gz"; + std::cout << "[INFO] Creating tarball: " << tar_filename << "\n"; + + std::vector cmd; + cmd.push_back("tar"); + for (auto &ex : exclusions) { + cmd.push_back("--exclude=" + ex); + } + cmd.push_back("--exclude=.git/"); + cmd.push_back("-czf"); + cmd.push_back(tar_filename); + cmd.push_back(fs::path(source_dir).filename().string()); + + run_command(cmd, source_dir.parent_path()); + std::cout << "[INFO] Tarball created and compressed: " << tar_filename << "\n"; +} diff --git a/build-packages-cpp/common.h b/build-packages-cpp/common.h new file mode 100644 index 0000000..86dfc56 --- /dev/null +++ b/build-packages-cpp/common.h @@ -0,0 +1,10 @@ +#pragma once +#include +#include +#include +#include + +std::string parse_version(const std::filesystem::path &changelog_path); +void run_command(const std::vector &cmd, const std::optional &cwd = std::nullopt, bool show_output=false); +void clean_old_logs(const std::filesystem::path &log_dir, int max_age_seconds=86400); +void create_tarball(const std::string &name, const std::filesystem::path &source_dir, const std::vector &exclusions); diff --git a/build-packages-cpp/main.cpp b/build-packages-cpp/main.cpp new file mode 100644 index 0000000..11f4c22 --- /dev/null +++ b/build-packages-cpp/main.cpp @@ -0,0 +1,644 @@ +#include "common.h" +#include "update_maintainer.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace fs = std::filesystem; + +static const std::string BASE_DIR = "/srv/lubuntu-ci/repos"; +static const std::string DEBFULLNAME = "Lugito"; +static const std::string DEBEMAIL = "info@lubuntu.me"; +static const std::string OUTPUT_DIR = BASE_DIR + "/build_output"; +static const std::vector SUPPRESSED_LINTIAN_TAGS = { + "orig-tarball-missing-upstream-signature", + "package-has-long-file-name", + "adopted-extended-field" +}; +static const std::string BASE_OUTPUT_DIR = "/srv/lubuntu-ci/output"; +static const std::string LOG_DIR = BASE_OUTPUT_DIR + "/logs/source_builds"; +static std::string BASE_LINTIAN_DIR; +static const std::string REAL_LINTIAN_DIR = BASE_OUTPUT_DIR + "/lintian"; +static std::string urgency_level_override = "low"; +static int worker_count = 5; + +static std::ofstream log_file_stream; + +static void log_all(const std::string &msg, bool is_error=false) { + if (is_error) { + std::cerr << msg; + } else { + std::cout << msg; + } + if (log_file_stream.is_open()) { + log_file_stream << msg; + log_file_stream.flush(); + } +} + +static void log_info(const std::string &msg) { + log_all("[INFO] " + msg + "\n"); +} + +static void log_warning(const std::string &msg) { + log_all("[WARN] " + msg + "\n", false); +} + +static void log_error(const std::string &msg) { + log_all("[ERROR] " + msg + "\n", true); +} + +static void run_command_silent_on_success(const std::vector &cmd, const std::optional &cwd = std::nullopt) { + std::string full_cmd; + for (auto &c: cmd) full_cmd += c + " "; + std::string exec_cmd = full_cmd; + if(cwd) exec_cmd = "cd " + cwd->string() + " && " + exec_cmd; + + log_info("Executing: " + full_cmd); + + FILE* pipe = popen(exec_cmd.c_str(), "r"); + if(!pipe) { + log_error("Failed to run: " + full_cmd); + throw std::runtime_error("Command failed"); + } + std::stringstream ss; + { + char buffer[256]; + while(fgets(buffer,256,pipe)) { + ss << buffer; + } + } + int ret = pclose(pipe); + if (ret != 0) { + log_error("Command failed: " + full_cmd); + log_error("Output:\n" + ss.str()); + throw std::runtime_error("Command failed"); + } +} + +// Initialize libgit2 once +static void git_init_once() { + static std::once_flag flag; + std::call_once(flag, [](){ + git_libgit2_init(); + }); +} + +static void git_fetch_and_checkout(const fs::path &repo_path, const std::string &repo_url, const std::optional &branch) { + git_init_once(); + git_repository* repo = nullptr; + bool need_clone = false; + if(fs::exists(repo_path)) { + int err = git_repository_open(&repo, repo_path.string().c_str()); + if(err<0) { + log_warning("Cannot open repo at " + repo_path.string() + ", recloning"); + fs::remove_all(repo_path); + need_clone = true; + } + } else { + need_clone = true; + } + + if(!need_clone && repo!=nullptr) { + git_remote* remote = nullptr; + int err = git_remote_lookup(&remote, repo, "origin"); + if(err<0) { + log_warning("No origin remote? Recloning"); + git_repository_free(repo); + fs::remove_all(repo_path); + need_clone = true; + } else { + const char* url = git_remote_url(remote); + if(!url || repo_url!=url) { + log_info("Remote URL differs. Removing and recloning."); + git_remote_free(remote); + git_repository_free(repo); + fs::remove_all(repo_path); + need_clone = true; + } else { + // fetch + git_remote_free(remote); + git_remote* origin = nullptr; + git_remote_lookup(&origin, repo, "origin"); + git_fetch_options fetch_opts = GIT_FETCH_OPTIONS_INIT; + git_remote_fetch(origin, nullptr, &fetch_opts, nullptr); + git_remote_free(origin); + + if(branch) { + git_reference* ref = nullptr; + std::string fullbranch = "refs/remotes/origin/" + *branch; + if(git_reference_lookup(&ref, repo, fullbranch.c_str())==0) { + git_object* target = nullptr; + git_reference_peel(&target, ref, GIT_OBJECT_COMMIT); + git_checkout_options co_opts = GIT_CHECKOUT_OPTIONS_INIT; + co_opts.checkout_strategy = GIT_CHECKOUT_FORCE; + git_checkout_tree(repo, target, &co_opts); + git_reference_free(ref); + git_repository_set_head_detached(repo, git_object_id(target)); + git_object_free(target); + } else { + log_error("Branch " + *branch + " not found, recloning"); + git_repository_free(repo); + fs::remove_all(repo_path); + need_clone = true; + } + } + git_repository_free(repo); + } + } + } + + if(need_clone) { + git_clone_options clone_opts = GIT_CLONE_OPTIONS_INIT; + git_checkout_options co_opts = GIT_CHECKOUT_OPTIONS_INIT; + co_opts.checkout_strategy = GIT_CHECKOUT_FORCE; + clone_opts.checkout_opts = co_opts; + git_repository* newrepo = nullptr; + int err = git_clone(&newrepo, repo_url.c_str(), repo_path.string().c_str(), &clone_opts); + if(err<0) { + const git_error* e = git_error_last(); + log_error(std::string("Git clone failed: ")+(e?e->message:"unknown")); + throw std::runtime_error("Git clone failed"); + } + if(branch) { + git_reference* ref = nullptr; + std::string fullbranch = "refs/remotes/origin/" + *branch; + if(git_reference_lookup(&ref, newrepo, fullbranch.c_str())==0) { + git_object* target = nullptr; + git_reference_peel(&target, ref, GIT_OBJECT_COMMIT); + git_checkout_options co_opts = GIT_CHECKOUT_OPTIONS_INIT; + co_opts.checkout_strategy = GIT_CHECKOUT_FORCE; + git_checkout_tree(newrepo, target, &co_opts); + git_reference_free(ref); + git_repository_set_head_detached(newrepo, git_object_id(target)); + git_object_free(target); + } else { + log_error("Git checkout of branch " + *branch + " failed after clone."); + git_repository_free(newrepo); + throw std::runtime_error("Branch checkout failed"); + } + } + git_repository_free(newrepo); + } +} + +static YAML::Node load_config(const fs::path &config_path) { + YAML::Node config = YAML::LoadFile(config_path.string()); + if (!config["packages"] || !config["releases"]) { + throw std::runtime_error("Config file must contain 'packages' and 'releases' sections."); + } + return config; +} + +static void publish_lintian() { + if(!BASE_LINTIAN_DIR.empty() && fs::exists(BASE_LINTIAN_DIR)) { + for (auto &p : fs::recursive_directory_iterator(BASE_LINTIAN_DIR)) { + if (fs::is_regular_file(p)) { + fs::path rel = fs::relative(p.path(), BASE_LINTIAN_DIR); + fs::path dest = fs::path(REAL_LINTIAN_DIR) / rel; + fs::create_directories(dest.parent_path()); + std::error_code ec; + fs::copy_file(p.path(), dest, fs::copy_options::overwrite_existing, ec); + } + } + fs::remove_all(BASE_LINTIAN_DIR); + } +} + +// Define get_exclusions here before usage +static std::vector get_exclusions(const fs::path &packaging) { + std::vector exclusions; + fs::path cpr = packaging / "debian" / "copyright"; + if(!fs::exists(cpr)) return exclusions; + + std::ifstream f(cpr); + if(!f) return exclusions; + std::string line; + bool found = false; + while(std::getline(f,line)) { + if (line.find("Files-Excluded:") != std::string::npos) { + size_t pos=line.find(':'); + if(pos!=std::string::npos) { + std::string excl = line.substr(pos+1); + std::istringstream iss(excl); + std::string token; + while(iss>>token) { + exclusions.push_back(token); + } + } + break; + } + } + return exclusions; +} + +int main(int argc, char** argv) { + fs::create_directories(LOG_DIR); + fs::create_directories(OUTPUT_DIR); + + auto now = std::time(nullptr); + std::tm tm = *std::gmtime(&now); + char buf[32]; + std::strftime(buf, sizeof(buf), "%Y%m%dT%H%M%S", &tm); + std::string current_time = buf; + + std::string uuid_part = current_time.substr(0,8); + BASE_LINTIAN_DIR = BASE_OUTPUT_DIR + "/.lintian.tmp." + uuid_part; + fs::create_directories(BASE_LINTIAN_DIR); + + fs::path log_file = fs::path(LOG_DIR) / (current_time + ".log"); + log_file_stream.open(log_file); + if(!log_file_stream.is_open()) { + std::cerr<<"[ERROR] Unable to open log file.\n"; + return 1; + } + + bool skip_dput = false; + bool skip_cleanup = false; + std::string config_path; + for(int i=1; istd::optional{ + if(pkg["packaging_branch"] && pkg["packaging_branch"].IsScalar()) { + return pkg["packaging_branch"].as(); + } else if (releases.size()>0) { + return "ubuntu/" + releases[0].as(); + } + return std::nullopt; + }; + + auto parse_version = [&](const fs::path &changelog_path){ + std::ifstream f(changelog_path); + if(!f) throw std::runtime_error("Changelog not found: " + changelog_path.string()); + std::string first_line; + std::getline(f, first_line); + size_t start = first_line.find('('); + size_t end = first_line.find(')'); + if(start==std::string::npos||end==std::string::npos) throw std::runtime_error("Invalid changelog format"); + std::string version_match = first_line.substr(start+1,end-(start+1)); + std::string epoch; + std::string upstream_version = version_match; + if(auto pos=version_match.find(':'); pos!=std::string::npos) { + epoch = version_match.substr(0,pos); + upstream_version = version_match.substr(pos+1); + } + if(auto pos=upstream_version.find('-'); pos!=std::string::npos) { + upstream_version=upstream_version.substr(0,pos); + } + std::regex git_regex("(\\+git[0-9]+)?(~[a-z]+)?$"); + upstream_version = std::regex_replace(upstream_version, git_regex, ""); + auto t = std::time(nullptr); + std::tm tm = *std::gmtime(&t); + char buf[32]; + std::strftime(buf, sizeof(buf), "%Y%m%d%H%M", &tm); + std::string current_date = buf; + std::string version; + if(!epoch.empty()) { + version = epoch + ":" + upstream_version + "+git" + current_date; + } else { + version = upstream_version + "+git" + current_date; + } + return version; + }; + + auto run_source_lintian = [&](const std::string &name, const fs::path &source_path){ + log_info("Running Lintian for " + name); + fs::path temp_file = fs::temp_directory_path() / ("lintian_suppress_" + name + ".txt"); + { + std::ofstream of(temp_file); + for (auto &tag: SUPPRESSED_LINTIAN_TAGS) { + of<&1"; + FILE* pipe = popen(cmd.c_str(),"r"); + std::stringstream ss; + if(pipe) { + char buffer[256]; + while(fgets(buffer,256,pipe)) { + ss< &changes_files, const std::vector &devel_changes_files){ + if(!changes_files.empty()) { + std::string hr_changes; + for(auto &c: changes_files) hr_changes += c+" "; + log_info("Uploading "+hr_changes+"to "+upload_target+" using dput"); + std::vector cmd = {"dput",upload_target}; + for(auto &c: changes_files) cmd.push_back(c); + try { + run_command_silent_on_success(cmd, OUTPUT_DIR); + log_info("Completed upload of changes to "+upload_target); + for(auto &file: devel_changes_files) { + if(!file.empty()) { + run_source_lintian(name, file); + } + } + } catch (...) { + // error logged already + } + } + }; + + auto update_changelog = [&](const fs::path &packaging_dir, const std::string &release, const std::string &version_with_epoch){ + std::string name = packaging_dir.filename().string(); + log_info("Updating changelog for " + name + " to version " + version_with_epoch + "-0ubuntu1~ppa1"); + run_command_silent_on_success({"git","checkout","debian/changelog"}, packaging_dir); + std::vector cmd={ + "dch","--distribution",release,"--package",name,"--newversion",version_with_epoch+"-0ubuntu1~ppa1","--urgency",urgency_level_override,"CI upload." + }; + run_command_silent_on_success(cmd, packaging_dir); + }; + + auto build_package = [&](const fs::path &packaging_dir, const std::map &env_vars, bool large) { + std::string name = packaging_dir.filename().string(); + log_info("Building source package for " + name); + fs::path temp_dir; + if(large) { + temp_dir = fs::path(OUTPUT_DIR)/(".tmp_"+name+"_"+env_vars.at("VERSION")); + fs::create_directories(temp_dir); + log_warning(name+" is quite large and will not fit in /tmp, building at "+temp_dir.string()); + } else { + temp_dir = fs::temp_directory_path()/("tmp_build_"+name+"_"+env_vars.at("VERSION")); + fs::create_directories(temp_dir); + } + + std::error_code ec; + fs::path temp_packaging_dir = temp_dir/name; + fs::create_directories(temp_packaging_dir,ec); + fs::copy(packaging_dir/"debian", temp_packaging_dir/"debian", fs::copy_options::recursive, ec); + + std::string tarball_name = name+"_"+env_vars.at("VERSION")+".orig.tar.gz"; + fs::path tarball_source = fs::path(BASE_DIR)/(name+"_MAIN.orig.tar.gz"); + fs::path tarball_dest = temp_dir/tarball_name; + fs::copy_file(tarball_source,tarball_dest,fs::copy_options::overwrite_existing,ec); + + for (auto &e: env_vars) { + setenv(e.first.c_str(), e.second.c_str(),1); + } + + std::vector cmd_build={"debuild","--no-lintian","-S","-d","-sa","-nc"}; + try { + run_command_silent_on_success(cmd_build,temp_packaging_dir); + run_command_silent_on_success({"git","checkout","debian/changelog"}, packaging_dir); + } catch(...) { + fs::remove_all(temp_dir,ec); + throw; + } + + std::string pattern = name+"_"+env_vars.at("VERSION"); + for(auto &entry: fs::directory_iterator(temp_dir)) { + std::string fname=entry.path().filename().string(); + if(fname.rfind(pattern,0)==0) { + fs::path dest=fs::path(OUTPUT_DIR)/fname; + fs::copy_file(entry.path(),dest,fs::copy_options::overwrite_existing,ec); + log_info("Copied "+fname+" to "+OUTPUT_DIR); + } + } + + std::string changes_file; + for(auto &entry : fs::directory_iterator(OUTPUT_DIR)) { + std::string fname=entry.path().filename().string(); + if(fname.rfind(name+"_"+env_vars.at("VERSION"),0)==0 && fname.ends_with("_source.changes")) { + changes_file=entry.path().string(); + } + } + + fs::remove_all(temp_dir,ec); + + if(changes_file.empty()) { + log_error("No changes file found after build."); + throw std::runtime_error("Changes file not found"); + } + log_info("Built package, changes file: "+changes_file); + return changes_file; + }; + + auto process_package = [&](const YAML::Node &pkg){ + std::string name = pkg["name"] ? pkg["name"].as() : ""; + std::string upload_target = pkg["upload_target"] ? pkg["upload_target"].as() : "ppa:lubuntu-ci/unstable-ci-proposed"; + if(name.empty()) { + log_warning("Skipping package due to missing name."); + return; + } + fs::path packaging_destination = fs::path(BASE_DIR)/name; + fs::path changelog_path = packaging_destination/"debian"/"changelog"; + std::string version = parse_version(changelog_path); + + bool large = pkg["large"] ? pkg["large"].as() : false; + std::vector>> built_changes; + + std::string epoch; + std::string version_no_epoch=version; + if(auto pos=version.find(':');pos!=std::string::npos) { + epoch=version.substr(0,pos); + version_no_epoch=version.substr(pos+1); + } + + for (auto rel : releases) { + std::string release = rel.as(); + log_info("Building "+name+" for "+release); + + std::string release_version_no_epoch = version_no_epoch + "~" + release; + fs::path tarball_source = fs::path(BASE_DIR)/(name+"_MAIN.orig.tar.gz"); + fs::path tarball_dest = fs::path(BASE_DIR)/(name+"_"+release_version_no_epoch+".orig.tar.gz"); + fs::copy_file(tarball_source,tarball_dest,fs::copy_options::overwrite_existing); + + std::string version_for_dch = epoch.empty()? release_version_no_epoch : (epoch+":"+release_version_no_epoch); + + std::map env_map; + env_map["DEBFULLNAME"]=DEBFULLNAME; + env_map["DEBEMAIL"]=DEBEMAIL; + env_map["VERSION"]=release_version_no_epoch; + env_map["UPLOAD_TARGET"]=upload_target; + + try { + update_changelog(packaging_destination, release, version_for_dch); + std::string changes_file = build_package(packaging_destination, env_map, large); + if(!changes_file.empty()) { + built_changes.push_back({changes_file,env_map}); + } + } catch(std::exception &e) { + log_error("Error processing package '"+name+"' for release '"+release+"': "+std::string(e.what())); + } + + fs::remove(tarball_dest); + } + + std::vector changes_files; + for(auto &bc: built_changes) { + fs::path cf(bc.first); + changes_files.push_back(cf.filename().string()); + } + + std::unordered_set devel_changes_files; + if(releases.size()>0) { + std::string first_release = releases[0].as(); + for (auto &f: changes_files) { + if(f.find("~"+first_release)!=std::string::npos) { + devel_changes_files.insert((fs::path(OUTPUT_DIR)/f).string()); + } else { + devel_changes_files.insert(std::string()); + } + } + } + + if(built_changes.empty()) return; + + if(getenv("DEBFULLNAME")==nullptr) setenv("DEBFULLNAME",DEBFULLNAME.c_str(),1); + if(getenv("DEBEMAIL")==nullptr) setenv("DEBEMAIL",DEBEMAIL.c_str(),1); + + if(skip_dput) { + for (auto &file : devel_changes_files) { + if(!file.empty()) { + run_source_lintian(name,file); + } + } + } else { + std::string real_upload_target = built_changes[0].second.at("UPLOAD_TARGET"); + dput_source(name, real_upload_target, changes_files, std::vector(devel_changes_files.begin(), devel_changes_files.end())); + } + + fs::remove(fs::path(BASE_DIR)/(name+"_MAIN.orig.tar.gz")); + }; + + auto prepare_package = [&](const YAML::Node &pkg){ + std::string name = pkg["name"] ? pkg["name"].as() : ""; + if(name.empty()) { + log_warning("Skipping package due to missing name."); + return; + } + + std::string upstream_url = pkg["upstream_url"] ? pkg["upstream_url"].as() : ("https://github.com/lxqt/"+name+".git"); + fs::path upstream_destination = fs::path(BASE_DIR)/("upstream-"+name); + std::optional packaging_branch = get_packaging_branch(pkg); + std::string packaging_url = pkg["packaging_url"] ? pkg["packaging_url"].as() : ("https://git.lubuntu.me/Lubuntu/"+name+"-packaging.git"); + fs::path packaging_destination = fs::path(BASE_DIR)/name; + + try { + git_fetch_and_checkout(upstream_destination, upstream_url, std::nullopt); + } catch(...) { + log_error("Failed to prepare upstream repo for "+name); + return; + } + + try { + git_fetch_and_checkout(packaging_destination, packaging_url, packaging_branch); + } catch(...) { + log_error("Failed to prepare packaging repo for "+name); + return; + } + + try { + update_maintainer((packaging_destination/"debian").string(), false); + } catch(std::exception &e) { + log_warning("update_maintainer: "+std::string(e.what())+" for "+name); + } + + auto exclusions = get_exclusions(packaging_destination); + create_tarball(name, upstream_destination, exclusions); + + process_package(pkg); + }; + + std::vector> futures; + for(auto pkg: packages) { + futures.push_back(std::async(std::launch::async, prepare_package, pkg)); + } + + for(auto &fut: futures) { + try { + fut.get(); + } catch(std::exception &e) { + log_error(std::string("Task generated an exception: ")+e.what()); + } + } + + if(!skip_cleanup) { + fs::remove_all(OUTPUT_DIR); + } + log_info("Publishing Lintian output..."); + publish_lintian(); + clean_old_logs(fs::path(LOG_DIR)); + + log_info("Script completed successfully."); + return 0; +} diff --git a/build-packages-cpp/update_maintainer.cpp b/build-packages-cpp/update_maintainer.cpp new file mode 100644 index 0000000..01c3422 --- /dev/null +++ b/build-packages-cpp/update_maintainer.cpp @@ -0,0 +1,204 @@ +#include "update_maintainer.h" +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static const char* PREVIOUS_UBUNTU_MAINTAINERS[] = { + "ubuntu core developers ", + "ubuntu core developers ", + "ubuntu motu developers " +}; +static const char* UBUNTU_MAINTAINER = "Ubuntu Developers "; + +class MaintainerUpdateException : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +static std::optional find_control_file(const fs::path &debian_dir) { + fs::path control_in = debian_dir / "control.in"; + fs::path control = debian_dir / "control"; + if (fs::exists(control_in)) return control_in; + if (fs::exists(control)) return control; + return std::nullopt; +} + +static fs::path find_changelog_file(const fs::path &debian_dir) { + fs::path changelog = debian_dir / "changelog"; + if (!fs::exists(changelog)) { + throw MaintainerUpdateException("No changelog file found"); + } + return changelog; +} + +static bool xsbc_managed_by_rules(const fs::path &debian_dir) { + fs::path rules = debian_dir / "rules"; + if (!fs::exists(rules)) return false; + std::ifstream rf(rules); + std::string line; + while (std::getline(rf, line)) { + if (line.find("XSBC-Original-") != std::string::npos) { + return true; + } + } + return false; +} + +static std::string get_distribution(const fs::path &changelog_file) { + // parse first line of changelog: "package (version) dist; urgency=..." + // dist is the token after ')' + std::ifstream f(changelog_file); + if(!f) throw MaintainerUpdateException("Unable to open changelog."); + std::string first_line; + std::getline(f, first_line); + // Format: "pkg (ver) dist; urgency=..." + // find ') ' + size_t pos = first_line.find(')'); + if(pos == std::string::npos) throw MaintainerUpdateException("Invalid changelog format"); + // after ') ', next token is distribution until space + // skip ')' + pos++; + while(pos < first_line.size() && std::isspace((unsigned char)first_line[pos])) pos++; + // now read until space or ';' + size_t start = pos; + while(pos < first_line.size() && !std::isspace((unsigned char)first_line[pos]) && first_line[pos] != ';') pos++; + std::string dist = first_line.substr(start, pos - start); + // remove -proposed-updates etc + size_t dashpos = dist.find('-'); + if (dashpos != std::string::npos) { + dist = dist.substr(0, dashpos); + } + return dist; +} + +static std::string read_file(const fs::path &p) { + std::ifstream f(p); + if(!f) throw MaintainerUpdateException("Cannot read file: " + p.string()); + std::stringstream ss; + ss << f.rdbuf(); + return ss.str(); +} + +static void write_file(const fs::path &p, const std::string &content) { + std::ofstream f(p); + if(!f) throw MaintainerUpdateException("Cannot write file: " + p.string()); + f << content; +} + +static std::optional get_field(const std::string &content, const std::string &field_regex) { + std::regex r(field_regex, std::regex_constants::multiline); + std::smatch m; + if(std::regex_search(content, m, r)) { + return m[1].str(); + } + return std::nullopt; +} + +static std::string set_field(const std::string &content, const std::string &field_regex, const std::string &new_line) { + std::regex r(field_regex, std::regex_constants::multiline); + return std::regex_replace(content, r, new_line); +} + +static void update_maintainer_file(const fs::path &control_file, const std::string &distribution, bool verbose) { + std::string c = read_file(control_file); + + auto original_maintainer = get_field(c, "^Maintainer:\\s?(.*)$"); + if(!original_maintainer) { + throw MaintainerUpdateException("No Maintainer field found"); + } + + std::string om = *original_maintainer; + std::string om_lower = om; + for (auto &ch : om_lower) ch = (char)std::tolower((unsigned char)ch); + + // Check previous ubuntu maintainers + for (auto &pm : PREVIOUS_UBUNTU_MAINTAINERS) { + std::string pm_lower = pm; + for (auto &ch: pm_lower) ch=(char)std::tolower((unsigned char)ch); + if(pm_lower == om_lower) { + if(verbose) { + std::cout<<"The old maintainer was: "<", lower_om.size()-11) != std::string::npos) { + if(verbose) { + std::cout<<"The Maintainer email is ubuntu.com address. Doing nothing.\n"; + } + return; + } + } + + // Debian distributions: stable, testing, unstable, experimental + if(distribution=="stable"||distribution=="testing"||distribution=="unstable"||distribution=="experimental") { + if(verbose) { + std::cout<<"The package targets Debian. Doing nothing.\n"; + } + return; + } + + // set XSBC-Original-Maintainer if needed + auto orig_field = get_field(c, "^(?:[XSBC]*-)?Original-Maintainer:\\s?(.*)$"); + if(orig_field && verbose) { + std::cout<<"Overwriting original maintainer: "<< *orig_field <<"\n"; + } + + if(verbose) { + std::cout<<"The original maintainer is: "<< om <<"\n"; + std::cout<<"Resetting as: "< + +void update_maintainer(const std::string &debian_directory, bool verbose=false);