From 19f8fd9481ca0700794770dc6446f6f4151b15cb Mon Sep 17 00:00:00 2001 From: Simon Quigley Date: Tue, 17 Dec 2024 01:29:02 -0600 Subject: [PATCH] Try porting lintian-ppa to C++ --- cpp/CMakeLists.txt | 8 +- cpp/lintian-ppa.cpp | 547 ++++++++++++++++++++++++++++++++++++++++++++ lintian-ppa | 171 -------------- 3 files changed, 553 insertions(+), 173 deletions(-) create mode 100644 cpp/lintian-ppa.cpp delete mode 100755 lintian-ppa diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 885f61c..fdf3bae 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -35,7 +35,11 @@ add_executable(fetch-indexes fetch-indexes.cpp utilities.cpp) target_include_directories(fetch-indexes PRIVATE /srv/lubuntu-ci/repos/ci-tools/include/launchpadlib-cpp) target_link_libraries(fetch-indexes PRIVATE lubuntuci CURL::libcurl yaml-cpp::yaml-cpp ZLIB::ZLIB /srv/lubuntu-ci/repos/ci-tools/lib/liblaunchpad.so) -set_target_properties(lubuntuci build-packages fetch-indexes update-maintainer PROPERTIES +add_executable(lintian-ppa lintian-ppa.cpp) +target_include_directories(lintian-ppa PRIVATE /srv/lubuntu-ci/repos/ci-tools/include/launchpadlib-cpp) +target_link_libraries(lintian-ppa PRIVATE lubuntuci /srv/lubuntu-ci/repos/ci-tools/lib/liblaunchpad.so) + +set_target_properties(lubuntuci build-packages fetch-indexes update-maintainer lintian-ppa PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH "$ORIGIN/lib" ) @@ -44,7 +48,7 @@ install(TARGETS lubuntuci LIBRARY DESTINATION lib ) -install(TARGETS build-packages fetch-indexes update-maintainer +install(TARGETS build-packages fetch-indexes update-maintainer lintian-ppa RUNTIME DESTINATION . ) diff --git a/cpp/lintian-ppa.cpp b/cpp/lintian-ppa.cpp new file mode 100644 index 0000000..e2352cb --- /dev/null +++ b/cpp/lintian-ppa.cpp @@ -0,0 +1,547 @@ +// Copyright (C) 2024 Simon Quigley +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#include "common.h" +#include "launchpad.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +// Global variables for logging +std::mutex logMutex; +std::ofstream globalLogFile; + +// Function to log informational messages +void log_info_custom(const std::string &msg) { + std::lock_guard lock(logMutex); + if (globalLogFile.is_open()) { + auto now = std::chrono::system_clock::now(); + std::time_t now_c = std::chrono::system_clock::to_time_t(now); + char timebuf[20]; + std::strftime(timebuf, sizeof(timebuf), "%Y-%m-%d %H:%M:%S", std::gmtime(&now_c)); + globalLogFile << timebuf << " - INFO - " << msg << "\n"; + globalLogFile.flush(); + } +} + +// Function to log error messages +void log_error_custom(const std::string &msg) { + std::lock_guard lock(logMutex); + if (globalLogFile.is_open()) { + auto now = std::chrono::system_clock::now(); + std::time_t now_c = std::chrono::system_clock::to_time_t(now); + char timebuf[20]; + std::strftime(timebuf, sizeof(timebuf), "%Y-%m-%d %H:%M:%S", std::gmtime(&now_c)); + globalLogFile << timebuf << " - ERROR - " << msg << "\n"; + globalLogFile.flush(); + } +} + +// Simple thread pool implementation +class ThreadPool { +public: + ThreadPool(size_t maxThreads) : stopFlag(false) { + for (size_t i = 0; i < maxThreads; ++i) { + workers.emplace_back([this]() { + while (true) { + std::function task; + + { + std::unique_lock lock(this->queueMutex); + this->condition.wait(lock, [this]() { return this->stopFlag || !this->tasks.empty(); }); + if (this->stopFlag && this->tasks.empty()) + return; + task = std::move(this->tasks.front()); + this->tasks.pop(); + } + + task(); + } + }); + } + } + + // Submit a task to the pool + template + void enqueue(F&& f) { + { + std::lock_guard lock(queueMutex); + if (stopFlag) + throw std::runtime_error("Enqueue on stopped ThreadPool"); + tasks.emplace(std::forward(f)); + } + condition.notify_one(); + } + + // Destructor joins all threads + ~ThreadPool() { + { + std::lock_guard lock(queueMutex); + stopFlag = true; + } + condition.notify_all(); + for (std::thread &worker: workers) + worker.join(); + } + +private: + std::vector workers; + std::queue> tasks; + + std::mutex queueMutex; + std::condition_variable condition; + bool stopFlag; +}; + +// Function to parse command-line arguments +struct Arguments { + std::string user; + std::string ppa; + std::optional ppa2; + std::optional override_output; +}; + +Arguments parseArguments(int argc, char* argv[]) { + Arguments args; + int opt; + bool showHelp = false; + + static struct option long_options[] = { + {"user", required_argument, 0, 'u'}, + {"ppa", required_argument, 0, 'p'}, + {"ppa2", required_argument, 0, '2'}, + {"override-output", required_argument, 0, 'o'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0} + }; + + while ((opt = getopt_long(argc, argv, "u:p:2:o:h", long_options, nullptr)) != -1) { + switch (opt) { + case 'u': + args.user = optarg; + break; + case 'p': + args.ppa = optarg; + break; + case '2': + args.ppa2 = optarg; + break; + case 'o': + args.override_output = optarg; + break; + case 'h': + default: + std::cout << "Usage: " << argv[0] << " --user --ppa [--ppa2 ] [--override-output ]\n"; + exit(0); + } + } + + if (args.user.empty() || args.ppa.empty()) { + std::cerr << "Error: --user and --ppa are required arguments.\n"; + std::cout << "Usage: " << argv[0] << " --user --ppa [--ppa2 ] [--override-output ]\n"; + exit(1); + } + + return args; +} + +// Function to parse the Changes file and extract Source and Architecture +struct ChangesInfo { + std::string source; + std::string architecture; +}; + +std::optional parse_changes_file(const fs::path& changesPath) { + if (!fs::exists(changesPath)) { + log_error_custom("Changelog not found: " + changesPath.string()); + return std::nullopt; + } + + std::ifstream infile(changesPath); + if (!infile.is_open()) { + log_error_custom("Unable to open changelog: " + changesPath.string()); + return std::nullopt; + } + + ChangesInfo info; + std::string line; + while (std::getline(infile, line)) { + if (line.empty()) + break; // End of headers + if (line.find("Source:") == 0) { + info.source = line.substr(7); + // Trim whitespace + info.source.erase(0, info.source.find_first_not_of(" \t")); + } + if (line.find("Architecture:") == 0) { + info.architecture = line.substr(13); + // Trim whitespace + info.architecture.erase(0, info.architecture.find_first_not_of(" \t")); + } + } + + infile.close(); + + if (info.source.empty() || info.architecture.empty()) { + log_error_custom("Invalid changelog format in: " + changesPath.string()); + return std::nullopt; + } + + return info; +} + +// Function to run lintian and capture its output +std::optional run_lintian(const fs::path& changesPath) { + std::vector lintianCmd = {"lintian", "-EvIL", "+pedantic", changesPath.filename().string()}; + try { + // Redirect stdout and stderr to capture output + std::string command = "lintian -EvIL +pedantic \"" + changesPath.string() + "\""; + std::array buffer; + std::string result; + std::unique_ptr pipe(popen(command.c_str(), "r"), pclose); + if (!pipe) { + log_error_custom("Failed to run lintian command."); + return std::nullopt; + } + while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) { + result += buffer.data(); + } + return result; + } catch (...) { + log_error_custom("Exception occurred while running lintian."); + return std::nullopt; + } +} + +// Function to process a single changes file URL +void process_sources(const std::string& url, const fs::path& baseOutputDir, const fs::path& lintianTmpDir) { + // Generate a unique temporary directory + std::string uuid_str; + uuid_t uuid; + uuid_generate(uuid, uuid_str.data()); + uuid_str = ""; + for(int i = 0; i < 16; ++i) { + char buf[3]; + snprintf(buf, sizeof(buf), "%02x", uuid[i]); + uuid_str += buf; + } + std::string tmpdir = (baseOutputDir / ("lintian_tmp_" + uuid_str)).string(); + + // Create temporary directory + fs::create_directories(tmpdir); + + // Extract the changes file name from URL + std::string changes_file = url.substr(url.find_last_of('/') + 1); + + log_info_custom("Downloading " + changes_file + " via dget."); + + // Run dget -u in the temporary directory + std::vector dgetCmd = {"dget", "-u", url}; + try { + run_command(dgetCmd, tmpdir); + } catch (const std::exception& e) { + log_error_custom("dget command failed for URL: " + url); + fs::remove_all(tmpdir); + return; + } + + // Parse the Changes file + fs::path changesPath = fs::path(tmpdir) / changes_file; + auto changesInfoOpt = parse_changes_file(changesPath); + if (!changesInfoOpt.has_value()) { + fs::remove_all(tmpdir); + return; + } + + ChangesInfo changesInfo = changesInfoOpt.value(); + + // Handle Architecture field + std::string arch = changesInfo.architecture; + arch = std::regex_replace(arch, std::regex("all"), ""); + arch = std::regex_replace(arch, std::regex("_translations"), ""); + std::istringstream iss(arch); + std::string arch_clean; + iss >> arch_clean; + if (arch_clean.empty()) { + fs::remove_all(tmpdir); + return; + } + + log_info_custom("Running Lintian for " + changesInfo.source + " on " + arch_clean); + + // Run lintian and capture output + auto lintianOutputOpt = run_lintian(changesPath); + if (!lintianOutputOpt.has_value()) { + fs::remove_all(tmpdir); + return; + } + std::string lintianOutput = lintianOutputOpt.value(); + + // Write lintian output to lintian_tmp/source/.txt + fs::path outputPath = lintianTmpDir / changesInfo.source; + fs::create_directories(outputPath); + fs::path archOutputFile = outputPath / (arch_clean + ".txt"); + try { + writeFile(archOutputFile, lintianOutput); + } catch (const std::exception& e) { + log_error_custom("Failed to write lintian output for " + changesInfo.source + " on " + arch_clean); + } + + // Remove temporary directory + fs::remove_all(tmpdir); +} + +// Function to perform rsync-like copy +void rsync_copy(const fs::path& source, const fs::path& destination) { + try { + if (!fs::exists(destination)) { + fs::create_directories(destination); + } + for (const auto& entry : fs::recursive_directory_iterator(source)) { + const auto& path = entry.path(); + auto relativePath = fs::relative(path, source); + fs::path destPath = destination / relativePath; + + if (fs::is_symlink(path)) { + if (fs::exists(destPath) || fs::is_symlink(destPath)) { + fs::remove(destPath); + } + auto target = fs::read_symlink(path); + fs::create_symlink(target, destPath); + } else if (fs::is_directory(path)) { + fs::create_directories(destPath); + } else if (fs::is_regular_file(path)) { + fs::copy_file(path, destPath, fs::copy_options::overwrite_existing); + } + } + } catch (const std::exception& e) { + log_error_custom("rsync_copy failed from " + source.string() + " to " + destination.string() + ": " + e.what()); + } +} + +int main(int argc, char* argv[]) { + // Parse command-line arguments + Arguments args = parseArguments(argc, argv); + + // Set BASE_OUTPUT_DIR + std::string BASE_OUTPUT_DIR = "/srv/lubuntu-ci/output/"; + if (args.override_output.has_value()) { + BASE_OUTPUT_DIR = args.override_output.value(); + } + + // Set LOG_DIR + fs::path LOG_DIR = fs::path(BASE_OUTPUT_DIR) / "logs" / "lintian"; + fs::create_directories(LOG_DIR); + + // Create log file with current UTC timestamp + auto now = std::chrono::system_clock::now(); + std::time_t now_c = std::chrono::system_clock::to_time_t(now); + char timestamp[20]; + std::strftime(timestamp, sizeof(timestamp), "%Y%m%dT%H%M%S", std::gmtime(&now_c)); + fs::path logFilePath = LOG_DIR / (std::string(timestamp) + ".log"); + + // Open global log file + globalLogFile.open(logFilePath, std::ios::app); + if (!globalLogFile.is_open()) { + std::cerr << "Error: Unable to open log file: " << logFilePath << std::endl; + return 1; + } + + log_info_custom("Starting lintian-ppa."); + + // Authenticate with Launchpad + log_info_custom("Logging into Launchpad..."); + std::shared_ptr lp_opt = launchpad::login(); + if (!lp_opt) { + log_error_custom("Failed to authenticate with Launchpad."); + return 1; + } + log_info_custom("Logged into Launchpad."); + + launchpad lp = *lp_opt; + + // Access Ubuntu distribution and current series + auto ubuntu_opt = lp.distributions.find("ubuntu"); + if (ubuntu_opt == lp.distributions.end()) { + log_error_custom("Failed to retrieve 'ubuntu' distribution."); + return 1; + } + distribution ubuntu = ubuntu_opt->second; + distro_series current_series; + + // Assuming current_series is accessible; adjust based on actual Launchpad API + // Here, we assume ubuntu.current_series is a valid member + if (ubuntu.getCurrentSeries(current_series) != 0) { + log_error_custom("Failed to retrieve current series."); + return 1; + } + log_info_custom("Current series: " + current_series.name); + + // Retrieve user and PPA + auto user_opt = lp.people.find(args.user); + if (user_opt == lp.people.end()) { + log_error_custom("Failed to retrieve user: " + args.user); + return 1; + } + person user = user_opt->second; + + auto ppa_opt = user.getPPAByName(ubuntu, args.ppa); + if (!ppa_opt.has_value()) { + log_error_custom("Failed to retrieve PPA: " + args.ppa); + return 1; + } + archive ppa = ppa_opt.value(); + log_info_custom("Retrieved PPA: " + args.ppa); + + std::optional ppa2_opt; + if (args.ppa2.has_value()) { + auto ppa2_found = user.getPPAByName(ubuntu, args.ppa2.value()); + if (!ppa2_found.has_value()) { + log_error_custom("Failed to retrieve PPA2: " + args.ppa2.value()); + return 1; + } + ppa2_opt = ppa2_found.value(); + log_info_custom("Retrieved PPA2: " + args.ppa2.value()); + } + + // Set up lintian directories + fs::path lintianDir = fs::path(BASE_OUTPUT_DIR) / "lintian"; + fs::path lintianTmpDir; + { + std::string uuid_str; + uuid_t uuid_bytes; + uuid_generate(uuid_bytes); + char uuid_cstr[37]; + uuid_unparse(uuid_bytes, uuid_cstr); + uuid_str = std::string(uuid_cstr); + // Truncate UUID to first 8 characters + uuid_str = uuid_str.substr(0, 8); + lintianTmpDir = fs::path(BASE_OUTPUT_DIR) / ("lintian_tmp_" + uuid_str); + } + fs::create_directories(lintianDir); + fs::create_directories(lintianTmpDir); + + // Define rsync function (already implemented as rsync_copy) + + // Initialize ThreadPool with 5 threads + ThreadPool pool(5); + + // Mutex for managing the published sources iterator + std::mutex sourcesMutex; + + // Function to iterate over published sources and enqueue tasks + auto main_source_iter = [&](ThreadPool& poolRef, std::vector>& futures) { + // Path to .LAST_RUN file + fs::path lastRunFile = lintianDir / ".LAST_RUN"; + std::chrono::system_clock::time_point lastRunTime = std::chrono::system_clock::now() - std::chrono::hours(24*365); + + if (fs::exists(lastRunFile)) { + std::ifstream infile(lastRunFile); + if (infile.is_open()) { + std::string lastRunStr; + std::getline(infile, lastRunStr); + infile.close(); + std::tm tm = {}; + std::istringstream ss(lastRunStr); + ss >> std::get_time(&tm, "%Y-%m-%dT%H:%M:%S"); + if (!ss.fail()) { + lastRunTime = std::chrono::system_clock::from_time_t(timegm(&tm)); + log_info_custom("Last run time: " + lastRunStr); + } else { + log_error_custom("Invalid format in .LAST_RUN file."); + } + } + } else { + log_info_custom(".LAST_RUN file does not exist. Using default last run time."); + } + + // Update .LAST_RUN with current time + { + std::ofstream outfile(lastRunFile, std::ios::trunc); + if (outfile.is_open()) { + auto currentTime = std::chrono::system_clock::now(); + std::time_t currentTime_c = std::chrono::system_clock::to_time_t(currentTime); + char timebuf[20]; + std::strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%S", std::gmtime(¤tTime_c)); + outfile << timebuf; + outfile.close(); + log_info_custom("Updated .LAST_RUN with current time: " + std::string(timebuf)); + } else { + log_error_custom("Failed to update .LAST_RUN file."); + } + } + + // Iterate over published sources + auto publishedSources = ppa.getPublishedSources("Published", current_series.name); + for (const auto& source : publishedSources) { + for (const auto& build : source.getBuilds()) { + if (build.buildstate == "Successfully built") { + // Assuming build.datebuilt is a std::chrono::system_clock::time_point + if (build.datebuilt >= lastRunTime) { + // Enqueue the process_sources task + poolRef.enqueue([=]() { + process_sources(build.changesfile_url, fs::path(BASE_OUTPUT_DIR), lintianTmpDir); + }); + } + } + } + } + }; + + // Start main_source_iter in the thread pool + std::vector> futures; + pool.enqueue([&]() { main_source_iter(pool, futures); }); + + // Wait for all tasks to complete by destructing the pool + // The ThreadPool destructor will wait for all tasks to finish + // So no additional synchronization is needed here + + // After all tasks are done, perform rsync + log_info_custom("All lintian tasks completed. Syncing temporary lintian data to final directory."); + rsync_copy(lintianTmpDir, lintianDir); + + // Remove temporary lintian directory + fs::remove_all(lintianTmpDir); + + // Clean old logs + clean_old_logs(LOG_DIR, 86400); // 1 day in seconds, adjust as needed + + log_info_custom("Lintian-ppa processing completed successfully."); + + // Close the global log file + if (globalLogFile.is_open()) { + globalLogFile.close(); + } + + return 0; +} diff --git a/lintian-ppa b/lintian-ppa deleted file mode 100755 index ad82b5c..0000000 --- a/lintian-ppa +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2024 Simon Quigley -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import argparse -import logging -import os -import shutil -import subprocess -import tempfile -import uuid -from common import clean_old_logs -from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED -from datetime import datetime, timedelta, timezone -from debian.deb822 import Changes -from launchpadlib.launchpad import Launchpad -from pathlib import Path - -BASE_OUTPUT_DIR = "/srv/lubuntu-ci/output/" - -parser = argparse.ArgumentParser(description="") -parser.add_argument("--user", "-u", required=True) -parser.add_argument("--ppa", "-p", required=True) -parser.add_argument("--ppa2", "-p2") -parser.add_argument("--override-output", "-o") -args = parser.parse_args() - -if args.override_output: - BASE_OUTPUT_DIR = args.override_output - -LOG_DIR = os.path.join(BASE_OUTPUT_DIR, "logs/lintian/") - -os.makedirs(LOG_DIR, exist_ok=True) -current_time = datetime.utcnow().strftime("%Y%m%dT%H%M%S") -log_file = os.path.join(LOG_DIR, f"{current_time}.log") -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(levelname)s - %(message)s", - handlers=[ - logging.FileHandler(log_file) - ] -) -logger = logging.getLogger("TimeBasedLogger") - -launchpad = Launchpad.login_with("lintian-ppa", "production", version="devel") -ubuntu = launchpad.distributions["ubuntu"] -current_series = ubuntu.current_series - -user = launchpad.people[args.user] -ppa = user.getPPAByName(distribution=ubuntu, name=args.ppa) -if args.ppa2: - ppa2 = user.getPPAByName(distribution=ubuntu, name=args.ppa) - -lintian = os.path.join(BASE_OUTPUT_DIR, "lintian") -lintian_tmp = os.path.join(BASE_OUTPUT_DIR, f".lintian.tmp.{str(uuid.uuid4())[:8]}") -if not os.path.exists(lintian): - os.mkdir(lintian) -if os.path.exists(lintian_tmp): - shutil.rmtree(lintian_tmp) -os.mkdir(lintian_tmp) - -def rsync(source, destination): - src = Path(source) - dst = Path(destination) - dst.mkdir(parents=True, exist_ok=True) - - for item in src.iterdir(): - src_path = item - dst_path = dst / item.name - - if src_path.is_symlink(): - if dst_path.exists() or dst_path.is_symlink(): - dst_path.unlink() - os.symlink(os.readlink(src_path), dst_path) - elif src_path.is_dir(): - shutil.copytree(src_path, dst_path, symlinks=True, dirs_exist_ok=True) - else: - shutil.copy2(src_path, dst_path) - -def process_sources(url): - tmpdir = os.path.join(BASE_OUTPUT_DIR, f".lintian.tmp.{str(uuid.uuid4())[:8]}") - os.mkdir(tmpdir) - changes_file = url.split("/")[-1] - logging.info(f"Downloading {changes_file} and friends via dget") - dget_command = ["dget", "-u", url] - result = subprocess.run(dget_command, cwd=tmpdir, capture_output=True) - - with open(os.path.join(tmpdir, changes_file), "r") as f: - changes_obj = Changes(f) - source = changes_obj["Source"] - arch = changes_obj["Architecture"].replace("all", "").replace("_translations", "").split(" ")[0].strip() - - if arch == "": - return - - logging.info(f"Running Lintian for {source} on {arch}") - lintian_command = ["lintian", "-EvIL", "+pedantic", changes_file] - result = subprocess.run(lintian_command, cwd=tmpdir, capture_output=True) - stderr = result.stderr.decode("utf-8").strip() - stdout = result.stdout.decode("utf-8").strip() - - if stderr == stdout: - lintian_output = stderr - elif stderr != "" and stdout == "": - lintian_output = stderr - elif stderr == "" and stdout != "": - lintian_output = stdout - else: - lintian_output = f"{stderr}\n{stdout}" - - output_path = os.path.join(lintian_tmp, source) - os.makedirs(output_path, exist_ok=True) - - with open(os.path.join(output_path, f"{arch}.txt"), "w") as f: - f.write(lintian_output) - - shutil.rmtree(tmpdir) - -with ThreadPoolExecutor(max_workers=5) as executor: - futures = set() - - def main_source_iter(): - last_run_file = os.path.join(lintian, ".LAST_RUN") - last_run_datetime = datetime.now(timezone.utc) - timedelta(days=365) - if os.path.exists(last_run_file): - with open(last_run_file, "r") as file: - last_run_time = file.read().strip() - - last_run_datetime = datetime.fromisoformat(last_run_time) - last_run_datetime = last_run_datetime.replace(tzinfo=timezone.utc) - logging.info(f"Last run: {last_run_datetime}") - - with open(last_run_file, "w") as file: - current_time = datetime.now(timezone.utc).isoformat() - file.write(current_time) - - for source in ppa.getPublishedSources(status="Published", distro_series=current_series): - for build in source.getBuilds(): - if build.buildstate == "Successfully built" and build.datebuilt >= last_run_datetime: - futures.add(executor.submit(process_sources, build.changesfile_url)) - - futures.add(executor.submit(main_source_iter)) - - while futures: - done, not_done = wait(futures, return_when=FIRST_COMPLETED) - - for future in done: - try: - result = future.result() - except Exception as e: - logging.exception("Task generated an exception:") - finally: - futures.remove(future) - -rsync(lintian_tmp, lintian) -shutil.rmtree(lintian_tmp) -clean_old_logs(LOG_DIR) -logging.info("Done")