#!/usr/bin/python2.7 # Copyright (C) 2011, 2012 Canonical Ltd. # Author: Martin Pitt # Author: Brian Murray # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 3 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Generate a HTML report of current NBS binary packages from a checkrdepends # output directory from __future__ import print_function from collections import defaultdict import csv from optparse import OptionParser import os import sys import time from charts import make_chart, make_chart_header from utils import read_tag_file default_base = '/home/ubuntu-archive/mirror/ubuntu' rdeps_with_alternates = [] def parse_checkrdepends_file(path, pkgmap): '''Parse one checkrdepends file into the NBS map''' cur_component = None cur_arch = None with open(path) as f: for line in f: if line.startswith('-- '): (cur_component, cur_arch) = line.split('/', 1)[1].split()[:2] continue assert cur_component assert cur_arch rdep = line.strip().split()[0] pkgmap.setdefault(rdep, (cur_component, []))[1].append(cur_arch) def _pkg_removable(options, pkg, nbs, checked_v): '''Recursively check if package is removable. checked_v is the working set of already checked vertices, to avoid infinite loops. ''' checked_v.add(pkg) packages = {} for rdep in nbs.get(pkg, []): if rdep in checked_v: continue global rdeps_with_alternates # utilize a copy of the arches as nbs will be modified arches = list(nbs[pkg][rdep][1]) for arch in arches: alternate_available = False if arch == 'build': ptype = 'source' file = 'Sources' else: ptype = 'binary-%s' % arch file = 'Packages' key = '%s/dists/%s/%s/%s/%s.gz' % \ (options.archive_base, options.suite, nbs[pkg][rdep][0], ptype, file) if key not in packages: packages[key] = read_tag_file(key, rdep) stanzas = packages[key] for stanza in stanzas: if 'binary' in ptype: fields = ('Pre-Depends', 'Depends', 'Recommends') else: fields = ('Build-Depends', 'Build-Depends-Indep') for field in fields: if field not in stanza: continue if '|' not in stanza[field]: continue for or_dep in stanza[field].split(','): if '|' not in or_dep: continue alternatives = [dep.strip() for dep in or_dep.split('|')] if pkg not in alternatives: continue for dep in alternatives: if dep == pkg: continue if dep not in nbs: alternate_available = True break if alternate_available: nbs[pkg][rdep][1].remove(arch) if len(nbs[pkg][rdep][1]) == 0: rdeps_with_alternates.append(rdep) if rdep not in nbs and rdep not in rdeps_with_alternates: try: checked_v.remove(rdep) except KeyError: pass return False if not _pkg_removable(options, rdep, nbs, checked_v): try: checked_v.remove(rdep) except KeyError: pass return False return True def get_removables(options, nbs): '''Get set of removable packages. This includes packages with no rdepends and disconnected subgraphs, i. e. clusters of NBS packages which only depend on each other. ''' removable = set() for p in nbs: if p in removable: continue checked_v = set() if _pkg_removable(options, p, nbs, checked_v): # we only add packages which are nbs to removable removable.update([p for p in checked_v if p in nbs]) return removable def html_report(options, nbs, removables): '''Generate HTML report from NBS map.''' global rdeps_with_alternates print('''\ NBS packages %s

NBS: Binary packages not built from any source

Archive Administrator commands

Run this command to remove NBS packages which are not required any more:

''' % make_chart_header()) print('

remove-package -m NBS ' '-d %s -s %s -b -y %s

' % (options.distribution, options.suite, ' '.join(sorted(removables)))) print('''

Reverse dependencies

Reverse dependencies which are NBS themselves
NBS package which can be removed safely

''') reverse_nbs = defaultdict(list) # non_nbs_pkg -> [nbspkg1, ...] pkg_component = {} # non_nbs_pkg -> (component, component_class) for pkg in sorted(nbs): nbsmap = nbs[pkg] if pkg in removables: cls = 'removable' else: cls = 'normal' print('\n' % (cls, pkg), end="") for rdep in sorted(nbsmap): if rdep in rdeps_with_alternates: continue (component, arches) = nbsmap[rdep] if component in ('main', 'restricted'): component_cls = 'sup' else: component_cls = 'unsup' if rdep in nbs: if rdep in removables: cls = 'removable' else: cls = 'nbs' else: cls = 'normal' reverse_nbs[rdep].append(pkg) pkg_component[rdep] = (component, component_cls) print('', end='') print(' ' % (cls, rdep), end='') print('' % (component_cls, component), end='') print('' % ' '.join(arches)) print('''
%s
    %s%s%s

Packages which depend on NBS packages

''') def sort_rev_nbs(k1, k2): len_cmp = cmp(len(reverse_nbs[k1]), len(reverse_nbs[k2])) if len_cmp == 0: return cmp(k1, k2) else: return -len_cmp for pkg in sorted(reverse_nbs, cmp=sort_rev_nbs): print(' ' '') print('
%s%s' % ( pkg, pkg_component[pkg][1], pkg_component[pkg][0]), end="") print(" ".join(sorted(reverse_nbs[pkg])), end="") print('
') if options.csv_file is not None: print("

Over time

") print(make_chart( os.path.basename(options.csv_file), ["removable", "total"])) print('

Generated at %s.

' % time.strftime('%Y-%m-%d %H:%M:%S %Z', time.gmtime(options.time))) print('') def main(): parser = OptionParser( usage="%prog ", description="Generate an HTML report of current NBS binary packages.") parser.add_option('-B', '--archive-base', dest='archive_base', help=('archive base directory (default: %s)' % default_base), default=default_base) parser.add_option('-d', '--distribution', default='ubuntu') parser.add_option('-s', '--suite', default='groovy') parser.add_option( '--csv-file', help='record CSV time series data in this file') options, args = parser.parse_args() if len(args) != 1: parser.error("need a checkrdepends output directory") options.time = time.time() # pkg -> rdep_pkg -> (component, [arch1, arch2, ...]) nbs = defaultdict(dict) for f in os.listdir(args[0]): if f.startswith('.') or f.endswith('.html'): continue parse_checkrdepends_file(os.path.join(args[0], f), nbs[f]) #with open('/tmp/dot', 'w') as dot: # print('digraph {', file=dot) # print(' ratio 0.1', file=dot) # pkgnames = set(nbs) # for m in nbs.itervalues(): # pkgnames.update(m) # for n in pkgnames: # print(' %s [label="%s"' % (n.replace('-', '').replace('.', ''), n), # end="", file=dot) # if n in nbs: # print(', style="filled", fillcolor="lightblue"', end="", file=dot) # print(']', file=dot) # print(file=dot) # for pkg, map in nbs.iteritems(): # for rd in map: # print(' %s -> %s' % ( # pkg.replace('-', '').replace('.', ''), # rd.replace('-', '').replace('.', '')), file=dot) # print('}', file=dot) removables = get_removables(options, nbs) html_report(options, nbs, removables) if options.csv_file is not None: if sys.version < "3": open_mode = "ab" open_kwargs = {} else: open_mode = "a" open_kwargs = {"newline": ""} csv_is_new = not os.path.exists(options.csv_file) with open(options.csv_file, open_mode, **open_kwargs) as csv_file: # Field names deliberately hardcoded; any changes require # manually rewriting the output file. fieldnames = [ "time", "removable", "total", ] csv_writer = csv.DictWriter(csv_file, fieldnames) if csv_is_new: csv_writer.writeheader() csv_writer.writerow({ "time": int(options.time * 1000), "removable": len(removables), "total": len(nbs), }) if __name__ == '__main__': main()