From 1d91f8212dc301d1a1ee52d4982199039103b4fc Mon Sep 17 00:00:00 2001 From: Simon Quigley Date: Tue, 17 Dec 2024 21:44:46 -0600 Subject: [PATCH] Try to convert tarball creation to C++ --- cpp/CMakeLists.txt | 5 ++-- cpp/build-packages.cpp | 4 +-- cpp/common.cpp | 63 +++++++++++++++++++++++++++++++++--------- cpp/common.h | 2 +- 4 files changed, 56 insertions(+), 18 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 545ae2b..19e422a 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -14,12 +14,13 @@ pkg_check_modules(LIBGIT2 REQUIRED IMPORTED_TARGET libgit2) find_package(CURL REQUIRED) find_library(UUID_LIB uuid) find_package(ZLIB REQUIRED) +pkg_check_modules(LIBARCHIVE REQUIRED libarchive) include_directories(/srv/lubuntu-ci/repos/ci-tools/include/launchpadlib-cpp) add_library(lubuntuci SHARED common.cpp utilities.cpp) -target_include_directories(lubuntuci PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(lubuntuci PUBLIC yaml-cpp::yaml-cpp PRIVATE CURL::libcurl /srv/lubuntu-ci/repos/ci-tools/lib/liblaunchpad.so) +target_include_directories(lubuntuci PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${LIBARCHIVE_INCLUDE_DIRS}) +target_link_libraries(lubuntuci PUBLIC yaml-cpp::yaml-cpp PRIVATE CURL::libcurl ${LIBARCHIVE_LIBRARIES} /srv/lubuntu-ci/repos/ci-tools/lib/liblaunchpad.so) add_library(update_maintainer_lib STATIC update-maintainer-lib.cpp) target_include_directories(update_maintainer_lib PRIVATE /srv/lubuntu-ci/repos/ci-tools/include/launchpadlib-cpp) diff --git a/cpp/build-packages.cpp b/cpp/build-packages.cpp index e4d08f2..6d023de 100644 --- a/cpp/build-packages.cpp +++ b/cpp/build-packages.cpp @@ -355,7 +355,7 @@ static void publish_lintian() { static std::vector get_exclusions(const fs::path &packaging) { log_verbose("Retrieving exclusions from: " + packaging.string()); - std::vector exclusions; + std::vector exclusions = {".git"}; fs::path cpr = packaging / "debian" / "copyright"; if(!fs::exists(cpr)) { log_verbose("No copyright file found."); @@ -644,7 +644,7 @@ static void pull_package(Package &pkg, const YAML::Node &releases) { auto exclusions = get_exclusions(packaging_destination); log_info("Creating tarball for package: " + pkg.name); try { - create_tarball(pkg.name, upstream_destination, exclusions); + create_tarball(pkg.name + "_MAIN.tar.gz", upstream_destination.string(), exclusions); log_info("Tarball created for package: " + pkg.name); } catch(const std::exception &e) { log_error("Failed to create tarball for package " + pkg.name + ": " + e.what()); diff --git a/cpp/common.cpp b/cpp/common.cpp index 4eb021b..965a3aa 100644 --- a/cpp/common.cpp +++ b/cpp/common.cpp @@ -14,6 +14,8 @@ // along with this program. If not, see . #include "common.h" +#include +#include #include #include #include @@ -112,22 +114,57 @@ void clean_old_logs(const fs::path &log_dir, int max_age_seconds) { } } -void create_tarball(const std::string &name, const fs::path &source_dir, const std::vector &exclusions) { - std::string tar_filename = name + "_MAIN.orig.tar.gz"; - std::cout << "[INFO] Creating tarball: " << tar_filename << "\n"; +void create_tarball(const std::string& tarballPath, const std::string& directory, const std::vector& exclusions) { + namespace fs = std::filesystem; + std::cout << "[INFO] Creating tarball: " << tarballPath << std::endl; - std::vector cmd; - cmd.push_back("tar"); - for (auto &ex : exclusions) { - cmd.push_back("--exclude=" + ex); + struct archive* a = archive_write_new(); + struct archive_entry* entry = nullptr; + + // Initialize the tarball + archive_write_add_filter_gzip(a); + archive_write_set_format_pax_restricted(a); + if (archive_write_open_filename(a, tarballPath.c_str()) != ARCHIVE_OK) { + throw std::runtime_error("Could not open tarball for writing: " + std::string(archive_error_string(a))); + } + + for (const auto& file : fs::recursive_directory_iterator(directory)) { + const auto& path = file.path(); + auto relativePath = fs::relative(path, directory).string(); + + // Check if the path matches any exclusion + bool excluded = std::any_of(exclusions.begin(), exclusions.end(), [&relativePath](const std::string& exclusion) { + return relativePath.find(exclusion) != std::string::npos; + }); + + if (excluded) { + continue; + } + + if (!fs::is_directory(path)) { + // Add the file to the tarball + entry = archive_entry_new(); + archive_entry_set_pathname(entry, relativePath.c_str()); + archive_entry_set_size(entry, fs::file_size(path)); + archive_entry_set_filetype(entry, AE_IFREG); + archive_entry_set_perm(entry, static_cast(fs::status(path).permissions())); + + archive_write_header(a, entry); + + // Write file contents + std::ifstream fileStream(path, std::ios::binary); + std::vector buffer((std::istreambuf_iterator(fileStream)), std::istreambuf_iterator()); + archive_write_data(a, buffer.data(), buffer.size()); + + archive_entry_free(entry); + } } - cmd.push_back("--exclude=.git/"); - cmd.push_back("-czf"); - cmd.push_back(tar_filename); - cmd.push_back(fs::path(source_dir).filename().string()); - run_command(cmd, source_dir.parent_path()); - std::cout << "[INFO] Tarball created and compressed: " << tar_filename << "\n"; + // Finalize and clean up + archive_write_close(a); + archive_write_free(a); + + std::cout << "[INFO] Tarball created and compressed: " << tarballPath << std::endl; } std::string get_current_utc_time() { diff --git a/cpp/common.h b/cpp/common.h index b110bcb..75a3574 100644 --- a/cpp/common.h +++ b/cpp/common.h @@ -22,5 +22,5 @@ std::string parse_version(const std::filesystem::path &changelog_path); void run_command(const std::vector &cmd, const std::optional &cwd = std::nullopt, bool show_output=false); void clean_old_logs(const std::filesystem::path &log_dir, int max_age_seconds=86400); -void create_tarball(const std::string &name, const std::filesystem::path &source_dir, const std::vector &exclusions); +void create_tarball(const std::string& tarballPath, const std::string& directory, const std::vector& exclusions); std::string get_current_utc_time();