You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
369 lines
13 KiB
369 lines
13 KiB
#! /usr/bin/python
|
|
# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
|
|
# GNU Affero General Public License version 3.
|
|
|
|
from __future__ import print_function
|
|
|
|
from collections import defaultdict
|
|
import logging
|
|
import optparse
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
import apt_pkg
|
|
from launchpadlib.errors import HTTPError
|
|
from launchpadlib.launchpad import Launchpad
|
|
|
|
|
|
re_extract_src_version = re.compile(r"(\S+)\s*\((.*)\)")
|
|
|
|
|
|
class ArchiveCruftCheckerError(Exception):
|
|
"""ArchiveCruftChecker specific exception.
|
|
|
|
Mostly used to describe errors in the initialization of this object.
|
|
"""
|
|
|
|
|
|
class TagFileNotFound(Exception):
|
|
"""Raised when an archive tag file could not be found."""
|
|
|
|
|
|
class ArchiveCruftChecker:
|
|
"""Perform overall checks to identify and remove obsolete records.
|
|
|
|
Use initialize() method to validate passed parameters and build the
|
|
infrastructure variables. It will raise ArchiveCruftCheckerError if
|
|
something goes wrong.
|
|
"""
|
|
|
|
# XXX cprov 2006-05-15: the default archive path should come
|
|
# from the config.
|
|
def __init__(self, launchpad_instance='production',
|
|
distribution_name='ubuntu', suite=None,
|
|
archive_path='/srv/launchpad.net/ubuntu-archive'):
|
|
"""Store passed arguments.
|
|
|
|
Also initialize empty variables for storing preliminary results.
|
|
"""
|
|
self.launchpad = Launchpad.login_anonymously(
|
|
'archive-cruft-check', launchpad_instance)
|
|
self.distribution_name = distribution_name
|
|
self.suite = suite
|
|
self.archive_path = archive_path
|
|
# initialize a group of variables to store temporary results
|
|
# available versions of published sources
|
|
self.source_versions = {}
|
|
# available binaries produced by published sources
|
|
self.source_binaries = {}
|
|
# 'Not Build From Source' binaries
|
|
self.nbs = defaultdict(lambda: defaultdict(dict))
|
|
# published binary package names
|
|
self.bin_pkgs = defaultdict(list)
|
|
# Architecture specific binary packages
|
|
self.arch_any = defaultdict(lambda: "0")
|
|
# proposed NBS (before clean up)
|
|
self.dubious_nbs = defaultdict(lambda: defaultdict(set))
|
|
# NBS after clean up
|
|
self.real_nbs = defaultdict(lambda: defaultdict(set))
|
|
# definitive NBS organized for clean up
|
|
self.nbs_to_remove = []
|
|
|
|
@property
|
|
def components_and_di(self):
|
|
components_and_di = []
|
|
for component in self.components:
|
|
components_and_di.append(component)
|
|
components_and_di.append('%s/debian-installer' % (component))
|
|
return components_and_di
|
|
|
|
@property
|
|
def dist_archive(self):
|
|
return os.path.join(
|
|
self.archive_path, self.distro.name, 'dists', self.suite)
|
|
|
|
def gunzipTagFileContent(self, filename):
|
|
"""Gunzip the contents of passed filename.
|
|
|
|
Check filename presence, if not present in the filesystem,
|
|
raises ArchiveCruftCheckerError. Use an tempfile.mkstemp()
|
|
to store the uncompressed content. Invoke system available
|
|
gunzip`, raises ArchiveCruftCheckError if it fails.
|
|
|
|
This method doesn't close the file descriptor used and does not
|
|
remove the temporary file from the filesystem, those actions
|
|
are required in the callsite. (apt_pkg.TagFile is lazy)
|
|
|
|
Return a tuple containing:
|
|
* temp file descriptor
|
|
* temp filename
|
|
* the contents parsed by apt_pkg.TagFile()
|
|
"""
|
|
if not os.path.exists(filename):
|
|
raise TagFileNotFound("File does not exist: %s" % filename)
|
|
|
|
temp_fd, temp_filename = tempfile.mkstemp()
|
|
subprocess.check_call(['gunzip', '-c', filename], stdout=temp_fd)
|
|
|
|
os.lseek(temp_fd, 0, os.SEEK_SET)
|
|
temp_file = os.fdopen(temp_fd)
|
|
# XXX cprov 2006-05-15: maybe we need some sort of data integrity
|
|
# check at this point, and maybe keep the uncompressed file
|
|
# for debug purposes, let's see how it behaves in real conditions.
|
|
parsed_contents = apt_pkg.TagFile(temp_file)
|
|
|
|
return temp_file, temp_filename, parsed_contents
|
|
|
|
def processSources(self):
|
|
"""Process archive sources index.
|
|
|
|
Build source_binaries, source_versions and bin_pkgs lists.
|
|
"""
|
|
logging.debug("Considering Sources:")
|
|
for component in self.components:
|
|
filename = os.path.join(
|
|
self.dist_archive, "%s/source/Sources.gz" % component)
|
|
|
|
logging.debug("Processing %s" % filename)
|
|
try:
|
|
temp_fd, temp_filename, parsed_sources = (
|
|
self.gunzipTagFileContent(filename))
|
|
except TagFileNotFound as warning:
|
|
logging.warning(warning)
|
|
return
|
|
try:
|
|
for section in parsed_sources:
|
|
source = section.find("Package")
|
|
source_version = section.find("Version")
|
|
binaries = section.find("Binary")
|
|
for binary in [
|
|
item.strip() for item in binaries.split(',')]:
|
|
self.bin_pkgs[binary].append(source)
|
|
|
|
self.source_binaries[source] = binaries
|
|
self.source_versions[source] = source_version
|
|
finally:
|
|
# close fd and remove temporary file used to store
|
|
# uncompressed tag file content from the filesystem.
|
|
temp_fd.close()
|
|
os.unlink(temp_filename)
|
|
|
|
def buildNBS(self):
|
|
"""Build the group of 'not build from source' binaries"""
|
|
# Checks based on the Packages files
|
|
logging.debug("Building not built from source list (NBS):")
|
|
for component in self.components_and_di:
|
|
for architecture in self.architectures:
|
|
self.buildArchNBS(component, architecture)
|
|
|
|
def buildArchNBS(self, component, architecture):
|
|
"""Build NBS per architecture.
|
|
|
|
Store results in self.nbs, also build architecture specific
|
|
binaries group (stored in self.arch_any)
|
|
"""
|
|
filename = os.path.join(
|
|
self.dist_archive,
|
|
"%s/binary-%s/Packages.gz" % (component, architecture))
|
|
|
|
logging.debug("Processing %s" % filename)
|
|
try:
|
|
temp_fd, temp_filename, parsed_packages = (
|
|
self.gunzipTagFileContent(filename))
|
|
except TagFileNotFound as warning:
|
|
logging.warn(warning)
|
|
return
|
|
|
|
try:
|
|
for section in parsed_packages:
|
|
package = section.find('Package')
|
|
source = section.find('Source', "")
|
|
version = section.find('Version')
|
|
architecture = section.find('Architecture')
|
|
|
|
if source == "":
|
|
source = package
|
|
|
|
if source.find("(") != -1:
|
|
m = re_extract_src_version.match(source)
|
|
source = m.group(1)
|
|
version = m.group(2)
|
|
|
|
if package not in self.bin_pkgs:
|
|
self.nbs[source][package][version] = ""
|
|
|
|
if architecture != "all":
|
|
if apt_pkg.version_compare(
|
|
version, self.arch_any[package]) < 1:
|
|
self.arch_any[package] = version
|
|
finally:
|
|
# close fd and remove temporary file used to store uncompressed
|
|
# tag file content from the filesystem.
|
|
temp_fd.close()
|
|
os.unlink(temp_filename)
|
|
|
|
def addNBS(self, nbs_d, source, version, package):
|
|
"""Add a new entry in given organized nbs_d list
|
|
|
|
Ensure the package is still published in the suite before add.
|
|
"""
|
|
result = self.archive.getPublishedBinaries(
|
|
binary_name=package, exact_match=True, status='Published')
|
|
result = [bpph for bpph in result
|
|
if bpph.distro_arch_series_link in self.das_urls]
|
|
|
|
if result:
|
|
nbs_d[source][version].add(package)
|
|
|
|
def refineNBS(self):
|
|
""" Distinguish dubious from real NBS.
|
|
|
|
They are 'dubious' if the version numbers match and 'real'
|
|
if the versions don't match.
|
|
It stores results in self.dubious_nbs and self.real_nbs.
|
|
"""
|
|
for source in self.nbs:
|
|
for package in self.nbs[source]:
|
|
versions = sorted(
|
|
self.nbs[source][package], cmp=apt_pkg.version_compare)
|
|
latest_version = versions.pop()
|
|
|
|
source_version = self.source_versions.get(source, "0")
|
|
|
|
if apt_pkg.version_compare(latest_version,
|
|
source_version) == 0:
|
|
# We don't actually do anything with dubious_nbs for
|
|
# now, so let's not waste time computing it.
|
|
#self.addNBS(self.dubious_nbs, source, latest_version,
|
|
# package)
|
|
pass
|
|
else:
|
|
self.addNBS(self.real_nbs, source, latest_version,
|
|
package)
|
|
|
|
def outputNBS(self):
|
|
"""Properly display built NBS entries.
|
|
|
|
Also organize the 'real' NBSs for removal in self.nbs_to_remove
|
|
attribute.
|
|
"""
|
|
output = "Not Built from Source\n"
|
|
output += "---------------------\n\n"
|
|
|
|
nbs_keys = sorted(self.real_nbs)
|
|
|
|
for source in nbs_keys:
|
|
proposed_bin = self.source_binaries.get(
|
|
source, "(source does not exist)")
|
|
proposed_version = self.source_versions.get(source, "??")
|
|
output += (" * %s_%s builds: %s\n"
|
|
% (source, proposed_version, proposed_bin))
|
|
output += "\tbut no longer builds:\n"
|
|
versions = sorted(
|
|
self.real_nbs[source], cmp=apt_pkg.version_compare)
|
|
|
|
for version in versions:
|
|
packages = sorted(self.real_nbs[source][version])
|
|
|
|
for pkg in packages:
|
|
self.nbs_to_remove.append(pkg)
|
|
|
|
output += " o %s: %s\n" % (
|
|
version, ", ".join(packages))
|
|
|
|
output += "\n"
|
|
|
|
if self.nbs_to_remove:
|
|
print(output)
|
|
else:
|
|
logging.debug("No NBS found")
|
|
|
|
def run(self):
|
|
"""Initialize and build required lists of obsolete entries in archive.
|
|
|
|
Check integrity of passed parameters and store organised data.
|
|
The result list is the self.nbs_to_remove which should contain
|
|
obsolete packages not currently able to be built from again.
|
|
Another preliminary lists can be inspected in order to have better
|
|
idea of what was computed.
|
|
If anything goes wrong mid-process, it raises ArchiveCruftCheckError,
|
|
otherwise a list of packages to be removes is printed.
|
|
"""
|
|
try:
|
|
self.distro = self.launchpad.distributions[
|
|
self.distribution_name]
|
|
except KeyError:
|
|
raise ArchiveCruftCheckerError(
|
|
"Invalid distribution: '%s'" % self.distribution_name)
|
|
|
|
if not self.suite:
|
|
self.distroseries = self.distro.current_series
|
|
self.suite = self.distroseries.name
|
|
else:
|
|
try:
|
|
self.distroseries = self.distro.getSeries(
|
|
name_or_version=self.suite.split('-')[0])
|
|
except HTTPError:
|
|
raise ArchiveCruftCheckerError(
|
|
"Invalid suite: '%s'" % self.suite)
|
|
|
|
if not os.path.exists(self.dist_archive):
|
|
raise ArchiveCruftCheckerError(
|
|
"Invalid archive path: '%s'" % self.dist_archive)
|
|
|
|
self.archive = self.distro.main_archive
|
|
self.distroarchseries = list(self.distroseries.architectures)
|
|
self.das_urls = [das.self_link for das in self.distroarchseries]
|
|
self.architectures = [a.architecture_tag
|
|
for a in self.distroarchseries]
|
|
self.components = self.distroseries.component_names
|
|
|
|
apt_pkg.init()
|
|
self.processSources()
|
|
self.buildNBS()
|
|
self.refineNBS()
|
|
self.outputNBS()
|
|
|
|
|
|
def main():
|
|
parser = optparse.OptionParser()
|
|
|
|
parser.add_option(
|
|
"-l", "--launchpad", dest="launchpad_instance", default="production")
|
|
parser.add_option(
|
|
"-d", "--distro", dest="distro", default="ubuntu", help="check DISTRO")
|
|
parser.add_option(
|
|
"-s", "--suite", dest="suite", help="only act on SUITE")
|
|
parser.add_option(
|
|
"-n", "--no-action", dest="action", default=True, action="store_false",
|
|
help="unused compatibility option")
|
|
parser.add_option(
|
|
"-v", "--verbose", dest="verbose", default=False, action="store_true",
|
|
help="emit verbose debugging messages")
|
|
|
|
options, args = parser.parse_args()
|
|
|
|
if args:
|
|
archive_path = args[0]
|
|
else:
|
|
logging.error('Archive path is required')
|
|
return 1
|
|
|
|
if options.verbose:
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
|
|
checker = ArchiveCruftChecker(
|
|
launchpad_instance=options.launchpad_instance,
|
|
distribution_name=options.distro, suite=options.suite,
|
|
archive_path=archive_path)
|
|
checker.run()
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|