Lubuntu/ubuntu-dev-tools
3
0
Fork 0
mirror of https://git.launchpad.net/ubuntu-dev-tools synced 2025-03-12 23:51:08 +00:00
Code Issues Packages Projects Releases Wiki Activity
ubuntu-dev-tools/syncpackage
Stefano Rivera c46bb9ed88 Use ubuntutools.logger in syncpackage
2010-12-27 18:24:21 +02:00

470 lines
18 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2008-2010 Martin Pitt <martin.pitt@canonical.com>,
# 2010 Benjamin Drung <bdrung@ubuntu.com>,
# 2010 Stefano Rivera <stefanor@ubuntu.com>
#
# ##################################################################
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# See file /usr/share/common-licenses/GPL-3 for more details.
#
# ##################################################################
import debian.deb822
import debian.debian_support
import hashlib
import optparse
import os
import re
import shutil
import subprocess
import sys
import urllib
from ubuntutools.requestsync.mail import getDebianSrcPkg \
as requestsync_mail_getDebianSrcPkg
from ubuntutools.requestsync.lp import getDebianSrcPkg, getUbuntuSrcPkg
from ubuntutools.logger import Logger
from ubuntutools.lp import udtexceptions
from ubuntutools.lp.lpapicache import Launchpad
class File(object):
def __init__(self, url, checksum, size):
self.url = url
self.name = os.path.basename(url)
self.checksum = checksum
self.size = size
def __repr__(self):
return self.name + " (" + self.checksum + " " + self.size + \
") source " + str(bool(self.is_source_file()))
def __eq__(self, other):
return self.name == other.name and self.checksum == other.checksum and \
self.size == other.size
def get_name(self):
return self.name
def is_source_file(self):
return re.match(".*\.orig.*\.tar\..*", self.name)
def download(self):
'''Download file (by URL) to the current directory.
If the file is already present, this function does nothing.'''
file_exists = os.path.exists(self.name)
if file_exists:
# Check for correct checksum
md5 = hashlib.md5()
md5.update(open(self.name).read())
file_exists = md5.hexdigest() == self.checksum
if not file_exists:
Logger.info('Downloading %s...', self.url)
try:
urllib.urlretrieve(self.url, self.name)
except IOError as err:
Logger.error('Failed to download %s [Errno %i]: %s.',
self.name, err.errno, err.strerror)
sys.exit(1)
class Version(debian.debian_support.Version):
def strip_epoch(self):
'''Removes the epoch from a Debian version string.
strip_epoch(1:1.52-1) will return "1.52-1" and strip_epoch(1.1.3-1) will
return "1.1.3-1".'''
parts = self.full_version.split(':')
if len(parts) > 1:
del parts[0]
version_without_epoch = ':'.join(parts)
return version_without_epoch
def get_related_debian_version(self):
related_debian_version = self.full_version
uidx = related_debian_version.find('ubuntu')
if uidx > 0:
related_debian_version = related_debian_version[:uidx]
uidx = related_debian_version.find('build')
if uidx > 0:
related_debian_version = related_debian_version[:uidx]
return Version(related_debian_version)
def is_modified_in_ubuntu(self):
return self.full_version.find('ubuntu') > 0
def remove_signature(dscname):
'''Removes the signature from a .dsc file if the .dsc file is signed.'''
dsc_file = open(dscname)
if dsc_file.readline().strip() == "-----BEGIN PGP SIGNED MESSAGE-----":
unsigned_file = []
# search until begin of body found
for line in dsc_file:
if line.strip() == "":
break
# search for end of body
for line in dsc_file:
if line.strip() == "":
break
unsigned_file.append(line)
dsc_file.close()
dsc_file = open(dscname, "w")
dsc_file.writelines(unsigned_file)
dsc_file.close()
def dsc_getfiles(dscurl):
'''Return list of files in a .dsc file (excluding the .dsc file itself).'''
basepath = os.path.dirname(dscurl)
dsc = debian.deb822.Dsc(urllib.urlopen(dscurl))
if 'Files' not in dsc:
Logger.error('No Files field found in the dsc file. Please check %s!',
os.path.basename(dscurl))
sys.exit(1)
files = []
for source_file in dsc['Files']:
url = os.path.join(basepath, source_file['name'])
if not source_file['name'].endswith('.dsc'):
files.append(File(url, source_file['md5sum'], source_file['size']))
return files
def add_fixed_bugs(changes, bugs):
'''Add additional Launchpad bugs to the list of fixed bugs in changes
file.'''
changes = [l for l in changes.split("\n") if l.strip() != ""]
# Remove duplicates
bugs = set(bugs)
for i in xrange(len(changes)):
if changes[i].startswith("Launchpad-Bugs-Fixed:"):
bugs.update(changes[i][22:].strip().split(" "))
changes[i] = "Launchpad-Bugs-Fixed: %s" % (" ".join(bugs))
break
elif i == len(changes) - 1:
# Launchpad-Bugs-Fixed entry does not exist in changes file
line = "Launchpad-Bugs-Fixed: %s" % (" ".join(bugs))
changes.append(line)
return "\n".join(changes + [""])
def sync_dsc(dscurl, debian_dist, release, name, email, bugs, keyid=None):
assert dscurl.endswith(".dsc")
dscname = os.path.basename(dscurl)
basepath = os.path.dirname(dscurl)
(srcpkg, new_ver) = dscname.split('_')
uploader = name + " <" + email + ">"
if os.path.exists(os.path.join(basepath, dscname)):
dscfile = dscurl
else:
try:
urllib.urlretrieve(dscurl, dscname)
except IOError as error:
Logger.error('Failed to download %s [Errno %i]: %s.',
dscname, error.errno, error.strerror)
sys.exit(1)
dscfile = debian.deb822.Dsc(file(dscname))
if "Version" not in dscfile:
Logger.error('No Version field found in the dsc file. Please check %s!',
dscname)
sys.exit(1)
new_ver = Version(dscfile["Version"])
try:
ubuntu_source = getUbuntuSrcPkg(srcpkg, release)
ubuntu_ver = Version(ubuntu_source.getVersion())
ubuntu_dsc = [f for f in ubuntu_source.sourceFileUrls()
if f.endswith(".dsc")]
assert len(ubuntu_dsc) == 1
ubuntu_dsc = ubuntu_dsc[0]
except udtexceptions.PackageNotFoundException:
ubuntu_ver = Version('~')
ubuntu_dsc = None
# No need to continue if version is not greater than current one
if new_ver <= ubuntu_ver:
Logger.error('%s version %s is not greater than already available %s',
srcpkg, new_ver, ubuntu_ver)
sys.exit(1)
Logger.debug('Source %s: current version %s, new version %s',
srcpkg, ubuntu_ver, new_ver)
files = dsc_getfiles(dscurl)
source_files = [f for f in files if f.is_source_file()]
Logger.debug('Files: %s', str([x.get_name() for x in files]))
Logger.debug('Source files: %s', str([x.get_name() for x in source_files]))
[f.download() for f in files]
if ubuntu_dsc is None:
ubuntu_files = None
else:
ubuntu_files = dsc_getfiles(ubuntu_dsc)
# do we need the orig.tar.gz?
need_orig = True
fakesync_files = []
if ubuntu_ver.upstream_version == new_ver.upstream_version:
# We need to check if all .orig*.tar.* tarballs exist in Ubuntu
need_orig = False
for source_file in source_files:
ubuntu_file = [f for f in ubuntu_files
if f.get_name() == source_file.get_name()]
if len(ubuntu_file) == 0:
# The source file does not exist in Ubuntu
Logger.info('%s does not exist in Ubuntu.',
source_file.get_name())
need_orig = True
elif not ubuntu_file[0] == source_file:
# The checksum of the files mismatch -> We need a fake sync
Logger.warn('The checksum of the file %s mismatch. '
'A fake sync is required.', source_file.get_name())
fakesync_files.append(ubuntu_file[0])
Logger.debug('Ubuntu version: %s', ubuntu_file[0])
Logger.debug('Debian version: %s', source_file)
Logger.debug('Needs source tarball: %s', str(need_orig))
cur_ver = ubuntu_ver.get_related_debian_version()
if ubuntu_ver.is_modified_in_ubuntu():
Logger.warn('Overwriting modified Ubuntu version %s, '
'setting current version to %s',
ubuntu_ver.full_version, cur_ver.full_version)
# extract package
cmd = ['dpkg-source', '-x', dscname]
env = os.environ
env['DEB_VENDOR'] = 'Ubuntu'
if not Logger.verbose:
cmd.insert(1, "-q")
Logger.command(cmd)
subprocess.check_call(cmd, env=env)
# Do a fake sync if required
if len(fakesync_files) > 0:
# Download Ubuntu files (override Debian source tarballs)
[f.download() for f in fakesync_files]
# change into package directory
directory = srcpkg + '-' + new_ver.upstream_version
Logger.command(('cd', directory))
os.chdir(directory)
# read Debian distribution from debian/changelog if not specified
if debian_dist is None:
line = open("debian/changelog").readline()
debian_dist = line.split(" ")[2].strip(";")
if len(fakesync_files) == 0:
# create the changes file
changes_filename = "%s_%s_source.changes" % \
(srcpkg, new_ver.strip_epoch())
cmd = ["dpkg-genchanges", "-S", "-v" + cur_ver.full_version,
"-DDistribution=" + release,
"-DOrigin=debian/" + debian_dist,
"-e" + uploader]
if need_orig:
cmd.append("-sa")
else:
cmd.append("-sd")
if not Logger.verbose:
cmd += ["-q"]
Logger.command(cmd + ['>', '../' + changes_filename])
changes = subprocess.Popen(cmd, stdout=subprocess.PIPE,
env={"DEB_VENDOR": "Ubuntu"}).communicate()[0]
# Add additional bug numbers
if len(bugs) > 0:
changes = add_fixed_bugs(changes, bugs)
# remove extracted (temporary) files
Logger.command(('cd', '..'))
os.chdir('..')
shutil.rmtree(directory, True)
# write changes file
changes_file = open(changes_filename, "w")
changes_file.writelines(changes)
changes_file.close()
# remove signature and sign package
remove_signature(dscname)
if keyid is not False:
cmd = ["debsign", changes_filename]
if not keyid is None:
cmd.insert(1, "-k" + keyid)
Logger.command(cmd)
subprocess.check_call(cmd)
else:
# Create fakesync changelog entry
new_ver = Version(new_ver.full_version + "fakesync1")
changes_filename = "%s_%s_source.changes" % \
(srcpkg, new_ver.strip_epoch())
if len(bugs) > 0:
message = "Fake sync due to mismatching orig tarball (LP: %s)." % \
(", ".join(["#" + str(b) for b in bugs]))
else:
message = "Fake sync due to mismatching orig tarball."
cmd = ["dch", "-v", new_ver.full_version, "-D", release, message]
env = {"DEBFULLNAME": name, "DEBEMAIL": email}
Logger.command(cmd)
subprocess.check_call(cmd, env=env)
# update the Maintainer field
cmd = ["update-maintainer"]
if not Logger.verbose:
cmd.append("-q")
Logger.command(cmd)
subprocess.check_call(cmd)
# Build source package
cmd = ["debuild", "--no-lintian", "-S", "-v" + cur_ver.full_version]
env = os.environ
env['DEB_VENDOR'] = 'Ubuntu'
if need_orig:
cmd += ['-sa']
if keyid:
cmd += ["-k" + keyid]
Logger.command(cmd)
returncode = subprocess.call(cmd, env=env)
if returncode != 0:
Logger.error('Source-only build with debuild failed. '
'Please check build log above.')
sys.exit(1)
def get_debian_dscurl(package, dist, release, version=None, component=None):
if dist is None:
dist = "unstable"
if type(version) == str:
version = Version(version)
if version is None or component is None:
debian_srcpkg = getDebianSrcPkg(package, dist)
try:
src_pkg = getUbuntuSrcPkg(package, release)
ubuntu_version = Version(src_pkg.getVersion())
except udtexceptions.PackageNotFoundException:
ubuntu_version = Version('~')
if ubuntu_version >= Version(debian_srcpkg.getVersion()):
# The LP importer is maybe out of date
debian_srcpkg = requestsync_mail_getDebianSrcPkg(package, dist)
if version is None:
version = Version(debian_srcpkg.getVersion())
if component is None:
component = debian_srcpkg.getComponent()
assert component in ("main", "contrib", "non-free")
if package.startswith("lib"):
group = package[0:4]
else:
group = package[0]
dsc_file = package + "_" + version.strip_epoch() + ".dsc"
dscurl = os.path.join("http://ftp.debian.org/debian/pool", component, group,
package, dsc_file)
return dscurl
def main():
usage = "%prog [options] <.dsc URL/path or package name>"
epilog = "See %s(1) for more info." % os.path.basename(sys.argv[0])
parser = optparse.OptionParser(usage=usage, epilog=epilog)
parser.add_option("-d", "--distribution", type="string",
dest="dist", default=None,
help="Debian distribution to sync from.")
parser.add_option("-r", "--release", dest="release", default=None,
help="Specify target Ubuntu release.")
parser.add_option("-V", "--debian-version", dest="debversion", default=None,
help="Specify the version to sync from.")
parser.add_option("-c", "--component", dest="component", default=None,
help="Specify the Debian component to sync from.")
parser.add_option("-v", "--verbose", help="print more information",
dest="verbose", action="store_true", default=False)
parser.add_option("-n", "--uploader-name", dest="uploader_name",
help="Use UPLOADER_NAME as the name of the maintainer "
"for this upload instead of evaluating DEBFULLNAME.",
default=None)
parser.add_option("-e", "--uploader-email", dest="uploader_email",
help="Use UPLOADER_EMAIL as email address of the "
"maintainer for this upload instead of evaluating "
"DEBEMAIL.", default=None)
parser.add_option("-k", "--key", dest="keyid", default=None,
help="Specify the key ID to be used for signing.")
parser.add_option('--dont-sign', dest='keyid', action='store_false',
help='Do not sign the upload')
parser.add_option("-b", "--bug", metavar="BUG",
help="Mark a Launchpad bug as being fixed by this upload",
dest="bugs", action="append", default=list())
(options, args) = parser.parse_args()
if len(args) == 0:
parser.error('No .dsc URL/path or package name specified.')
if len(args) > 1:
parser.error('Multiple .dsc URLs/paths or package names specified: '
+ ', '.join(args))
invalid_bug_numbers = [bug for bug in options.bugs if not bug.isdigit()]
if len(invalid_bug_numbers) > 0:
parser.error('Invalid bug number(s) specified: '
+ ', '.join(invalid_bug_numbers))
if options.uploader_name is None:
if "DEBFULLNAME" in os.environ:
options.uploader_name = os.environ["DEBFULLNAME"]
else:
parser.error('No uploader name specified. You must pass the '
'--uploader-name option or set the DEBFULLNAME '
'environment variable.')
if options.uploader_email is None:
if "DEBEMAIL" in os.environ:
options.uploader_email = os.environ["DEBEMAIL"]
else:
parser.error('No uploader email address specified. You must pass '
'the --uploader-email option or set the DEBEMAIL '
'environment variable.')
Launchpad.login_anonymously()
if options.release is None:
options.release = Launchpad.distributions["ubuntu"].current_series.name
if args[0].endswith(".dsc"):
dscurl = args[0]
else:
if options.component not in (None, "main", "contrib", "non-free"):
parser.error('%s is not a valid Debian component. '
'It should be one of main, contrib, or non-free.'
% options.component)
dscurl = get_debian_dscurl(args[0], options.dist, options.release,
options.debversion, options.component)
Logger.verbose = options.verbose
Logger.debug('.dsc url: %s', dscurl)
sync_dsc(dscurl, options.dist, options.release, options.uploader_name,
options.uploader_email, options.bugs, options.keyid)
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink