Lubuntu/ci-tools
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework tarball logic further

Browse Source
This commit is contained in:
Simon Quigley 2025-02-07 20:38:52 -06:00
parent 77cbfb46cf
commit 881adad442
1 changed files with 150 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

291
cpp/utilities.cpp
View File

@ -416,187 +416,196 @@ std::vector<std::string> extract_files_excluded(const std::string& filepath) {
} }
// Function to create a tarball // Function to create a tarball
void create_tarball(const std::string& tarballPath, const std::string& directory, const std::vector<std::string>& exclusions, std::shared_ptr<Log> log) { void create_tarball(const std::string &tarball_path,
log->append("Creating tarball: " + tarballPath); const std::string &directory,
const std::vector<std::string> &exclusions,
std::shared_ptr<Log> log) {
if (log)
log->append("Creating tarball: " + tarball_path);
try { try {
struct archive* a = archive_write_new(); // Ensure the source directory exists and is a directory.
if (!a) throw std::runtime_error("Failed to create a new archive."); if (!fs::exists(directory) || !fs::is_directory(directory)) throw std::runtime_error("Source directory does not exist or is not a directory: " + directory);
if (archive_write_add_filter_gzip(a) != ARCHIVE_OK) { // Use RAII for the archive writer.
struct ArchiveDeleter {
void operator()(struct archive *a) const {
if (a) archive_write_free(a);
}
};
std::unique_ptr<struct archive, ArchiveDeleter> a(archive_write_new());
if (!a) throw std::runtime_error("Failed to create a new archive writer.");
if (archive_write_add_filter_gzip(a.get()) != ARCHIVE_OK) {
std::string err = "Failed to add gzip filter: "; std::string err = "Failed to add gzip filter: ";
err += archive_error_string(a); err += archive_error_string(a.get());
archive_write_free(a);
throw std::runtime_error(err); throw std::runtime_error(err);
} }
if (archive_write_set_format_pax_restricted(a) != ARCHIVE_OK) { if (archive_write_set_format_pax_restricted(a.get()) != ARCHIVE_OK) {
std::string err = "Failed to set format: "; std::string err = "Failed to set archive format: ";
err += archive_error_string(a); err += archive_error_string(a.get());
archive_write_free(a);
throw std::runtime_error(err); throw std::runtime_error(err);
} }
if (archive_write_open_filename(a, tarballPath.c_str()) != ARCHIVE_OK) { if (archive_write_open_filename(a.get(), tarball_path.c_str()) != ARCHIVE_OK) {
std::string err = "Could not open tarball for writing: "; std::string err = "Could not open tarball for writing: ";
err += archive_error_string(a); err += archive_error_string(a.get());
archive_write_free(a);
throw std::runtime_error(err); throw std::runtime_error(err);
} }
// Iterate through the directory recursively without following symlinks // Iterate over the directory recursively.
for (auto it = fs::recursive_directory_iterator( for (auto it = fs::recursive_directory_iterator(directory, fs::directory_options::skip_permission_denied);
directory,
fs::directory_options::skip_permission_denied);
it != fs::recursive_directory_iterator(); ++it) { it != fs::recursive_directory_iterator(); ++it) {
const auto& path = it->path(); const auto &path = it->path();
try {
std::error_code ec;
fs::path relative_path = fs::relative(path, directory, ec); // Compute a relative path to the source directory.
if (ec) { std::error_code ec;
fs::path rel_path = fs::relative(path, directory, ec);
if (ec) {
if (log)
log->append("Failed to compute relative path for: " + path.string() + " Error: " + ec.message()); log->append("Failed to compute relative path for: " + path.string() + " Error: " + ec.message());
continue; continue;
} }
fs::path norm_rel_path = rel_path.lexically_normal();
std::string rel_path_str = norm_rel_path.string();
// Normalize the relative path to avoid discrepancies // Exclude files whose relative path contains any exclusion substring.
fs::path normalized_relative_path = relative_path.lexically_normal(); bool is_excluded = std::any_of(exclusions.begin(), exclusions.end(),
std::string relative_path_str = normalized_relative_path.string(); [&rel_path_str](const std::string &excl) {
return rel_path_str.find(excl) != std::string::npos;
});
if (is_excluded) continue;
// Exclusion logic (if any exclusions are provided) // Get file status without following symlinks.
bool excluded = std::any_of(exclusions.begin(), exclusions.end(), [&relative_path_str](const std::string& exclusion) { fs::file_status fstatus = it->symlink_status(ec);
return relative_path_str.find(exclusion) != std::string::npos; if (ec) {
}); if (log)
if (excluded) { continue; }
fs::file_status fstatus = it->symlink_status(ec);
if (ec) {
log->append("Failed to get file status for: " + path.string() + " Error: " + ec.message()); log->append("Failed to get file status for: " + path.string() + " Error: " + ec.message());
continue; continue;
} }
struct archive_entry* entry = archive_entry_new(); // Create a new archive entry.
if (!entry) { struct archive_entry *entry = archive_entry_new();
if (!entry) {
if (log)
log->append("Failed to create archive entry for: " + path.string()); log->append("Failed to create archive entry for: " + path.string());
archive_write_free(a); continue;
throw std::runtime_error("Failed to create archive entry."); }
}
std::string entry_path = relative_path_str; // For directories, ensure the entry path ends with a '/'
if (fs::is_directory(fstatus)) { std::string entry_path = rel_path_str;
// Ensure the directory pathname ends with '/' if (fs::is_directory(fstatus)) {
if (!entry_path.empty() && entry_path.back() != '/') { if (!entry_path.empty() && entry_path.back() != '/')
entry_path += '/'; entry_path.push_back('/');
} }
archive_entry_set_pathname(entry, entry_path.c_str()); archive_entry_set_pathname(entry, entry_path.c_str());
} else {
archive_entry_set_pathname(entry, entry_path.c_str());
}
// Set file type, permissions, and size // Set file type, size, and permissions.
if (fs::is_regular_file(fstatus)) { if (fs::is_regular_file(fstatus)) {
// Regular file uintmax_t filesize = fs::file_size(path, ec);
uintmax_t filesize = fs::file_size(path, ec);
if (ec) {
log->append("Cannot get file size for: " + path.string() + " Error: " + ec.message());
archive_entry_free(entry);
continue;
}
archive_entry_set_size(entry, static_cast<off_t>(filesize));
archive_entry_set_filetype(entry, AE_IFREG);
archive_entry_set_perm(entry, static_cast<mode_t>(fstatus.permissions()));
}
else if (fs::is_symlink(fstatus)) {
fs::path target = fs::read_symlink(path, ec);
if (ec) {
log->append("Cannot read symlink for: " + path.string() + " Error: " + ec.message());
archive_entry_free(entry);
continue;
}
archive_entry_set_symlink(entry, target.c_str());
archive_entry_set_filetype(entry, AE_IFLNK);
archive_entry_set_perm(entry, static_cast<mode_t>(fstatus.permissions()));
}
else if (fs::is_directory(fstatus)) {
archive_entry_set_size(entry, 0);
archive_entry_set_filetype(entry, AE_IFDIR);
archive_entry_set_perm(entry, static_cast<mode_t>(fstatus.permissions()));
}
else {
log->append("Unsupported file type for: " + path.string());
archive_entry_free(entry);
continue;
}
// Retrieve and set the modification time
fs::file_time_type ftime = fs::last_write_time(path, ec);
std::time_t mtime;
if (ec) { if (ec) {
log->append("Failed to get last write time for: " + path.string() + " Error: " + ec.message()); if (log)
// Obtain current UTC time as fallback log->append("Cannot get file size for: " + path.string() + " Error: " + ec.message());
auto now = std::chrono::system_clock::now();
mtime = std::chrono::system_clock::to_time_t(now);
log->append("Setting default mtime (current UTC time) for: " + path.string());
} else {
mtime = to_time_t(ftime);
}
archive_entry_set_mtime(entry, mtime, 0);
if (archive_write_header(a, entry) != ARCHIVE_OK) {
log->append("Failed to write header for: " + path.string() + " Error: " + archive_error_string(a));
archive_entry_free(entry); archive_entry_free(entry);
continue; continue;
} }
archive_entry_set_size(entry, static_cast<off_t>(filesize));
archive_entry_set_filetype(entry, AE_IFREG);
archive_entry_set_perm(entry, static_cast<mode_t>(fstatus.permissions()));
} else if (fs::is_directory(fstatus)) {
archive_entry_set_size(entry, 0);
archive_entry_set_filetype(entry, AE_IFDIR);
archive_entry_set_perm(entry, static_cast<mode_t>(fstatus.permissions()));
} else if (fs::is_symlink(fstatus)) {
fs::path target = fs::read_symlink(path, ec);
if (ec) {
if (log)
log->append("Cannot read symlink for: " + path.string() + " Error: " + ec.message());
archive_entry_free(entry);
continue;
}
archive_entry_set_symlink(entry, target.c_str());
archive_entry_set_filetype(entry, AE_IFLNK);
archive_entry_set_perm(entry, static_cast<mode_t>(fstatus.permissions()));
} else {
if (log)
log->append("Unsupported file type for: " + path.string());
archive_entry_free(entry);
continue;
}
if (fs::is_regular_file(fstatus)) { // Set the modification time.
std::ifstream fileStream(path, std::ios::binary); fs::file_time_type ftime = fs::last_write_time(path, ec);
if (!fileStream) { std::time_t mtime;
if (ec) {
if (log)
log->append("Failed to get last write time for: " + path.string() + " Error: " + ec.message());
auto now = std::chrono::system_clock::now();
mtime = std::chrono::system_clock::to_time_t(now);
if (log)
log->append("Setting current UTC time as modification time for: " + path.string());
} else {
mtime = to_time_t(ftime);
}
archive_entry_set_mtime(entry, mtime, 0);
// Write the header for the entry.
if (archive_write_header(a.get(), entry) != ARCHIVE_OK) {
if (log)
log->append("Failed to write header for: " + path.string() + " Error: " + archive_error_string(a.get()));
archive_entry_free(entry);
continue;
}
// If this is a regular file, write its contents.
if (fs::is_regular_file(fstatus)) {
std::ifstream in_file(path, std::ios::binary);
if (!in_file) {
if (log)
log->append("Failed to open file for reading: " + path.string()); log->append("Failed to open file for reading: " + path.string());
archive_entry_free(entry); archive_entry_free(entry);
continue; continue;
} }
const std::size_t buffer_size = 8192;
const std::size_t bufferSize = 8192; char buffer[buffer_size];
char buffer[bufferSize]; while (in_file) {
while (fileStream) { in_file.read(buffer, buffer_size);
fileStream.read(buffer, bufferSize); std::streamsize bytes_read = in_file.gcount();
std::streamsize bytesRead = fileStream.gcount(); if (bytes_read > 0) {
if (bytesRead > 0) { if (archive_write_data(a.get(), buffer, static_cast<size_t>(bytes_read)) < 0) {
if (archive_write_data(a, buffer, static_cast<size_t>(bytesRead)) < 0) { if (log)
log->append("Failed to write data for: " + path.string() + " Error: " + archive_error_string(a)); log->append("Failed to write data for: " + path.string() + " Error: " + archive_error_string(a.get()));
break; break;
}
} }
} }
if (fileStream.bad()) {
log->append("Error reading file: " + path.string());
}
} }
if (in_file.bad() && log)
archive_entry_free(entry); log->append("Error reading file: " + path.string());
} catch (const std::exception &e) {
log->append("Failed to add the following file to the tarball: " + path.string());
log->append(e.what());
} }
// Finish writing the entry.
if (archive_write_finish_entry(a.get()) != ARCHIVE_OK) {
if (log)
log->append("Failed to finish entry for: " + path.string() + " Error: " + archive_error_string(a.get()));
}
// No need to call archive_entry_free() as finish_entry() frees the entry.
} }
if (archive_write_close(a) != ARCHIVE_OK) { if (archive_write_close(a.get()) != ARCHIVE_OK) {
std::string err = "Failed to close archive: "; std::string err = "Failed to close archive: ";
err += archive_error_string(a); err += archive_error_string(a.get());
archive_write_free(a);
throw std::runtime_error(err);
}
if (archive_write_free(a) != ARCHIVE_OK) {
std::string err = "Failed to free archive: ";
err += archive_error_string(a);
throw std::runtime_error(err); throw std::runtime_error(err);
} }
// The unique_ptr will call archive_write_free() automatically.
} catch (const std::exception &e) { } catch (const std::exception &e) {
log->append("Failed to create tarball: " + tarballPath); if (log) {
log->append(e.what()); log->append("Failed to create tarball: " + tarball_path);
log->append(e.what());
}
return;
} }
log->append("Tarball created and compressed: " + tarballPath); if (log)
log->append("Tarball created and compressed: " + tarball_path);
} }