You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
428 lines
15 KiB
428 lines
15 KiB
#!/usr/bin/python3
|
|
|
|
# Parse removals.txt file
|
|
# Copyright (C) 2004, 2005, 2009, 2010, 2011, 2012 Canonical Ltd.
|
|
# Authors: James Troup <james.troup@canonical.com>,
|
|
# Colin Watson <cjwatson@ubuntu.com>
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
############################################################################
|
|
|
|
# What kind of genius logs 3.5 years of removal data in
|
|
# semi-human-parseable text and nothing else? Hmm. That would be me. :(
|
|
|
|
# MAAAAAAAAAAAAAADNESSSSSSSSSSSSSSSSS
|
|
|
|
############################################################################
|
|
|
|
from __future__ import print_function
|
|
|
|
import copy
|
|
import optparse
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
|
|
try:
|
|
from urllib.request import urlopen, urlretrieve
|
|
except ImportError:
|
|
from urllib import urlretrieve
|
|
from urllib2 import urlopen
|
|
import re
|
|
import logging
|
|
import gzip
|
|
import datetime
|
|
import subprocess
|
|
|
|
import apt_pkg
|
|
from launchpadlib.launchpad import Launchpad
|
|
|
|
import lputils
|
|
|
|
|
|
CONSUMER_KEY = "process-removals"
|
|
|
|
|
|
Debian = None
|
|
|
|
|
|
def parse_removals_file(options, removals_file):
|
|
if options.report_ubuntu_delta and options.no_action:
|
|
me = None
|
|
else:
|
|
me = options.launchpad.me.name
|
|
|
|
state = "first separator"
|
|
removal = {}
|
|
for line in removals_file:
|
|
line = line.strip().decode('UTF-8')
|
|
# skip over spurious empty lines
|
|
if not line and state in ("first separtor", "next separator", "date"):
|
|
continue
|
|
if line == ("=" * 73):
|
|
if state == "done":
|
|
state = "next separator"
|
|
elif state in ("first separator", "next separator"):
|
|
state = "date"
|
|
else:
|
|
raise RuntimeError("found separator but state is %s." % state)
|
|
# NB: The 'Reason' is an abbreviation, check any referenced bugs for
|
|
# more
|
|
elif state == "next separator" and line.startswith("NB:"):
|
|
state = "first separator"
|
|
# [Date: Tue, 9 Jan 2001 20:46:15 -0500] [ftpmaster: James Troup]
|
|
elif state in ("date", "next separator"):
|
|
try:
|
|
(date, ftpmaster, unused) = line.split("]")
|
|
date = date.replace("[Date: ", "")
|
|
state = "removed from"
|
|
except:
|
|
state = "broken date"
|
|
# Sat, 06 May 2017 08:45:30 +0000] [ftpmaster: Chris Lamb]
|
|
elif state == "broken date":
|
|
(date, ftpmaster, unused2) = line.split("]")
|
|
state = "removed from"
|
|
# Removed the following packages from unstable:
|
|
elif state == "removed from":
|
|
# complete processing of the date from the preceding line
|
|
try:
|
|
removal["date"] = time.mktime(time.strptime(
|
|
date, "%a, %d %b %Y %H:%M:%S %z"))
|
|
except ValueError:
|
|
removal["date"] = time.mktime(time.strptime(
|
|
date, "%a %d %b %H:%M:%S %Z %Y"))
|
|
removal["ftpmaster"] = ftpmaster.replace("] [ftpmaster: ", "")
|
|
|
|
prefix = "Removed the following packages from "
|
|
if not line.startswith(prefix):
|
|
raise RuntimeError(
|
|
"state = %s, expected '%s', got '%s'" %
|
|
(state, prefix, line))
|
|
line = line.replace(prefix, "")[:-1]
|
|
line = line.replace(" and", "")
|
|
line = line.replace(",", "")
|
|
removal["suites"] = line.split()
|
|
state = "before packages"
|
|
elif state == "before packages" or state == "after packages":
|
|
if line:
|
|
raise RuntimeError(
|
|
"state = %s, expected '', got '%s'" % (state, line))
|
|
if state == "before packages":
|
|
state = "packages"
|
|
elif state == "after packages":
|
|
state = "reason prefix"
|
|
# xroach | 4.0-8 | source, alpha, arm, hppa, i386, ia64, m68k, \
|
|
# mips, mipsel, powerpc, s390, sparc
|
|
# Closed bugs: 158188
|
|
elif state == "packages":
|
|
if line.find("|") != -1:
|
|
package, version, architectures = [
|
|
word.strip() for word in line.split("|")]
|
|
architectures = [
|
|
arch.strip() for arch in architectures.split(",")]
|
|
removal.setdefault("packages", [])
|
|
removal["packages"].append([package, version, architectures])
|
|
elif line.startswith("Closed bugs: "):
|
|
line = line.replace("Closed bugs: ", "")
|
|
removal["closed bugs"] = line.split()
|
|
state = "after packages"
|
|
elif not line:
|
|
state = "reason prefix"
|
|
else:
|
|
raise RuntimeError(
|
|
"state = %s, expected package list or 'Closed bugs:', "
|
|
"got '%s'" % (state, line))
|
|
# ------------------- Reason -------------------
|
|
elif state == "reason prefix":
|
|
expected = "------------------- Reason -------------------"
|
|
if not line == expected:
|
|
raise RuntimeError(
|
|
"state = %s, expected '%s', got '%s'" %
|
|
(state, expected, line))
|
|
state = "reason"
|
|
# RoSRM; license problems.
|
|
# ----------------------------------------------
|
|
elif state == "reason":
|
|
if line == "----------------------------------------------":
|
|
state = "done"
|
|
do_removal(me, options, removal)
|
|
removal = {}
|
|
else:
|
|
removal.setdefault("reason", "")
|
|
removal["reason"] += "%s\n" % line
|
|
|
|
# nothing should go here
|
|
|
|
|
|
def show_reverse_depends(options, package):
|
|
"""Show reverse dependencies.
|
|
|
|
This is mostly done by calling reverse-depends, but with some tweaks to
|
|
make the output easier to read in context.
|
|
"""
|
|
series_name = options.series.name
|
|
commands = (
|
|
["reverse-depends", "-r", series_name, "src:%s" % package],
|
|
["reverse-depends", "-r", series_name, "-a", "source", "src:%s" % package],
|
|
)
|
|
for command in commands:
|
|
subp = subprocess.Popen(command, stdout=subprocess.PIPE)
|
|
line = None
|
|
for line in subp.communicate()[0].splitlines():
|
|
line = line.decode('UTF-8').rstrip("\n")
|
|
if line == "No reverse dependencies found":
|
|
line = None
|
|
else:
|
|
print(line)
|
|
if line:
|
|
print()
|
|
|
|
|
|
non_meta_re = re.compile(r'^[a-zA-Z0-9+,./:=@_-]+$')
|
|
|
|
|
|
def shell_escape(arg):
|
|
if non_meta_re.match(arg):
|
|
return arg
|
|
else:
|
|
return "'%s'" % arg.replace("'", "'\\''")
|
|
|
|
|
|
seen_sources = set()
|
|
|
|
|
|
def do_removal(me, options, removal):
|
|
if options.removed_from and options.removed_from not in removal["suites"]:
|
|
return
|
|
if options.date and int(options.date) > int(removal["date"]):
|
|
return
|
|
if options.architecture:
|
|
for architecture in re.split(r'[, ]+', options.architecture):
|
|
package_list = copy.copy(removal["packages"])
|
|
for entry in package_list:
|
|
(package, version, architectures) = entry
|
|
if architecture not in architectures:
|
|
removal["packages"].remove(entry)
|
|
if not removal["packages"]:
|
|
return
|
|
|
|
for entry in removal.get("packages", []):
|
|
(package, version, architectures) = entry
|
|
if options.source and options.source != package:
|
|
continue
|
|
if package in seen_sources:
|
|
continue
|
|
seen_sources.add(package)
|
|
if package in Debian and not (options.force and options.source):
|
|
logging.info("%s (%s): back in sid - skipping.", package, version)
|
|
continue
|
|
|
|
spphs = []
|
|
for pocket in ("Release", "Proposed"):
|
|
options.pocket = pocket
|
|
if pocket == "Release":
|
|
options.suite = options.series.name
|
|
else:
|
|
options.suite = "%s-proposed" % options.series.name
|
|
try:
|
|
spphs.append(
|
|
lputils.find_latest_published_source(options, package))
|
|
except lputils.PackageMissing:
|
|
pass
|
|
|
|
if not spphs:
|
|
logging.debug("%s (%s): not found", package, version)
|
|
continue
|
|
|
|
if options.report_ubuntu_delta:
|
|
if 'ubuntu' in version:
|
|
print("%s %s" % (package, version))
|
|
continue
|
|
if options.no_action:
|
|
continue
|
|
|
|
for spph in spphs:
|
|
logging.info(
|
|
"%s (%s/%s), removed from Debian on %s",
|
|
package, version, spph.source_package_version,
|
|
time.strftime("%Y-%m-%d", time.localtime(removal["date"])))
|
|
|
|
removal["reason"] = removal["reason"].rstrip('.\n')
|
|
try:
|
|
bugs = ';' + ','.join(
|
|
[" Debian bug #%s" % item for item in removal["closed bugs"]])
|
|
except KeyError:
|
|
bugs = ''
|
|
reason = "(From Debian) " + removal["reason"] + bugs
|
|
|
|
subprocess.call(['seeded-in-ubuntu', package])
|
|
show_reverse_depends(options, package)
|
|
for spph in spphs:
|
|
if options.no_action:
|
|
print("remove-package -s %s%s -e %s -m %s %s" % (
|
|
options.series.name,
|
|
"-proposed" if spph.pocket == "Proposed" else "",
|
|
spph.source_package_version, shell_escape(reason),
|
|
package))
|
|
else:
|
|
from ubuntutools.question import YesNoQuestion
|
|
print("Removing packages:")
|
|
print("\t%s" % spph.display_name)
|
|
for binary in spph.getPublishedBinaries():
|
|
print("\t\t%s" % binary.display_name)
|
|
print("Comment: %s" % reason)
|
|
if YesNoQuestion().ask("Remove", "no") == "yes":
|
|
spph.requestDeletion(removal_comment=reason)
|
|
|
|
|
|
def fetch_removals_file(options):
|
|
removals = "removals-full.txt"
|
|
if options.date:
|
|
thisyear = datetime.datetime.today().year
|
|
if options.date >= time.mktime(time.strptime(str(thisyear), "%Y")):
|
|
removals = "removals.txt"
|
|
logging.debug("Fetching %s" % removals)
|
|
return urlopen("http://ftp-master.debian.org/%s" % removals)
|
|
|
|
|
|
def read_sources():
|
|
global Debian
|
|
Debian = {}
|
|
base = "http://ftp.debian.org/debian"
|
|
|
|
logging.debug("Reading Debian sources")
|
|
for component in "main", "contrib", "non-free":
|
|
filename = "Debian_unstable_%s_Sources" % component
|
|
if not os.path.exists(filename):
|
|
url = "%s/dists/unstable/%s/source/Sources.gz" % (base, component)
|
|
logging.info("Fetching %s" % url)
|
|
fetched, headers = urlretrieve(url)
|
|
out = open(filename, 'wb')
|
|
gzip_handle = gzip.GzipFile(fetched)
|
|
while True:
|
|
block = gzip_handle.read(65536)
|
|
if not block:
|
|
break
|
|
out.write(block)
|
|
out.close()
|
|
gzip_handle.close()
|
|
sources_filehandle = open(filename)
|
|
Sources = apt_pkg.TagFile(sources_filehandle)
|
|
while Sources.step():
|
|
pkg = Sources.section.find("Package")
|
|
version = Sources.section.find("Version")
|
|
|
|
if (pkg in Debian and
|
|
apt_pkg.version_compare(Debian[pkg]["version"],
|
|
version) > 0):
|
|
continue
|
|
|
|
Debian[pkg] = {}
|
|
Debian[pkg]["version"] = version
|
|
sources_filehandle.close()
|
|
|
|
|
|
def parse_options():
|
|
parser = optparse.OptionParser()
|
|
|
|
parser.add_option("-l", "--launchpad", dest="launchpad_instance",
|
|
default="production")
|
|
|
|
parser.add_option("-a", "--architecture", metavar="ARCH",
|
|
help="remove from ARCH")
|
|
parser.add_option("-d", "--date", metavar="DATE",
|
|
help="only those removed since DATE (Unix epoch or %Y-%m-%d)")
|
|
|
|
parser.add_option("-r", "--removed-from", metavar="SUITE",
|
|
help="only those removed from SUITE (aka distroseries)")
|
|
|
|
parser.add_option("-s", "--source", metavar="NAME",
|
|
help="only source package NAME")
|
|
parser.add_option("--force", action="store_true", default=False,
|
|
help="force removal even for packages back in unstable "
|
|
"(only with -s)")
|
|
|
|
parser.add_option("-f", "--filename",
|
|
help="parse FILENAME")
|
|
|
|
parser.add_option("-n", "--no-action", action="store_true",
|
|
help="don't remove packages; just print remove-package "
|
|
"commands")
|
|
parser.add_option("-v", "--verbose", action="store_true",
|
|
help="emit verbose debugging messages")
|
|
|
|
parser.add_option("--report-ubuntu-delta", action="store_true",
|
|
help="skip and report packages with Ubuntu deltas")
|
|
|
|
options, args = parser.parse_args()
|
|
if len(args):
|
|
parser.error("no arguments expected")
|
|
|
|
if options.date:
|
|
if len(options.date) == 8:
|
|
print("The format of date should be %Y-%m-%d.")
|
|
sys.exit(1)
|
|
try:
|
|
int(options.date)
|
|
except ValueError:
|
|
options.date = time.mktime(time.strptime(options.date, "%Y-%m-%d"))
|
|
else:
|
|
options.date = time.mktime(time.strptime("2009-02-01", "%Y-%m-%d"))
|
|
if not options.removed_from:
|
|
options.removed_from = "unstable"
|
|
if not options.architecture:
|
|
options.architecture = "source"
|
|
|
|
if options.verbose:
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
else:
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
return options, args
|
|
|
|
|
|
def main():
|
|
apt_pkg.init()
|
|
|
|
"""Initialization, including parsing of options."""
|
|
options, args = parse_options()
|
|
|
|
if options.report_ubuntu_delta and options.no_action:
|
|
options.launchpad = Launchpad.login_anonymously(
|
|
CONSUMER_KEY, options.launchpad_instance)
|
|
else:
|
|
options.launchpad = Launchpad.login_with(
|
|
CONSUMER_KEY, options.launchpad_instance)
|
|
|
|
options.distribution = options.launchpad.distributions["ubuntu"]
|
|
options.series = options.distribution.current_series
|
|
options.archive = options.distribution.main_archive
|
|
options.version = None
|
|
|
|
read_sources()
|
|
|
|
if options.filename:
|
|
removals_file = open(options.filename, "rb")
|
|
else:
|
|
removals_file = fetch_removals_file(options)
|
|
parse_removals_file(options, removals_file)
|
|
removals_file.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|