Mirror support in syncpackage

This commit is contained in:
Stefano Rivera 2010-12-28 15:35:15 +02:00
parent 1b93ed57ed
commit 62073c1085
2 changed files with 74 additions and 78 deletions

View File

@ -22,7 +22,6 @@
import debian.deb822 import debian.deb822
import debian.debian_support import debian.debian_support
import hashlib
import optparse import optparse
import os import os
import re import re
@ -30,6 +29,7 @@ import shutil
import subprocess import subprocess
import sys import sys
import urllib import urllib
import urlparse
from ubuntutools.config import UDTConfig, ubu_email from ubuntutools.config import UDTConfig, ubu_email
from ubuntutools.requestsync.mail import getDebianSrcPkg \ from ubuntutools.requestsync.mail import getDebianSrcPkg \
@ -38,6 +38,7 @@ from ubuntutools.requestsync.lp import getDebianSrcPkg, getUbuntuSrcPkg
from ubuntutools.logger import Logger from ubuntutools.logger import Logger
from ubuntutools.lp import udtexceptions from ubuntutools.lp import udtexceptions
from ubuntutools.lp.lpapicache import Launchpad from ubuntutools.lp.lpapicache import Launchpad
from ubuntutools.mirrors import pull_source_pkg
class File(object): class File(object):
def __init__(self, url, checksum, size): def __init__(self, url, checksum, size):
@ -60,28 +61,6 @@ class File(object):
def is_source_file(self): def is_source_file(self):
return re.match(".*\.orig.*\.tar\..*", self.name) return re.match(".*\.orig.*\.tar\..*", self.name)
def download(self):
'''Download file (by URL) to the current directory.
If the file is already present, this function does nothing.'''
file_exists = os.path.exists(self.name)
if file_exists:
# Check for correct checksum
md5 = hashlib.md5()
md5.update(open(self.name).read())
file_exists = md5.hexdigest() == self.checksum
if not file_exists:
Logger.info('Downloading %s...', self.url)
try:
urllib.urlretrieve(self.url, self.name)
except IOError as err:
Logger.error('Failed to download %s [Errno %i]: %s.',
self.name, err.errno, err.strerror)
sys.exit(1)
class Version(debian.debian_support.Version): class Version(debian.debian_support.Version):
def strip_epoch(self): def strip_epoch(self):
@ -107,7 +86,7 @@ class Version(debian.debian_support.Version):
return Version(related_debian_version) return Version(related_debian_version)
def is_modified_in_ubuntu(self): def is_modified_in_ubuntu(self):
return self.full_version.find('ubuntu') > 0 return 'ubuntu' in self.full_version
def remove_signature(dscname): def remove_signature(dscname):
'''Removes the signature from a .dsc file if the .dsc file is signed.''' '''Removes the signature from a .dsc file if the .dsc file is signed.'''
@ -169,22 +148,14 @@ def add_fixed_bugs(changes, bugs):
return "\n".join(changes + [""]) return "\n".join(changes + [""])
def sync_dsc(dscurl, debian_dist, release, name, email, bugs, keyid=None): def sync_dsc(dscname, debian_dist, release, name, email, bugs, ubuntu_mirror,
assert dscurl.endswith(".dsc") keyid=None):
dscname = os.path.basename(dscurl) assert dscname.endswith(".dsc")
basepath = os.path.dirname(dscurl) assert os.path.exists(dscname)
assert '/' not in dscname
(srcpkg, new_ver) = dscname.split('_') (srcpkg, new_ver) = dscname.split('_')
uploader = name + " <" + email + ">" uploader = name + " <" + email + ">"
if os.path.exists(os.path.join(basepath, dscname)):
dscfile = dscurl
else:
try:
urllib.urlretrieve(dscurl, dscname)
except IOError as error:
Logger.error('Failed to download %s [Errno %i]: %s.',
dscname, error.errno, error.strerror)
sys.exit(1)
dscfile = debian.deb822.Dsc(file(dscname)) dscfile = debian.deb822.Dsc(file(dscname))
if "Version" not in dscfile: if "Version" not in dscfile:
Logger.error('No Version field found in the dsc file. Please check %s!', Logger.error('No Version field found in the dsc file. Please check %s!',
@ -203,19 +174,13 @@ def sync_dsc(dscurl, debian_dist, release, name, email, bugs, keyid=None):
ubuntu_ver = Version('~') ubuntu_ver = Version('~')
ubuntu_dsc = None ubuntu_dsc = None
# No need to continue if version is not greater than current one
if new_ver <= ubuntu_ver:
Logger.error('%s version %s is not greater than already available %s',
srcpkg, new_ver, ubuntu_ver)
sys.exit(1)
Logger.debug('Source %s: current version %s, new version %s', Logger.debug('Source %s: current version %s, new version %s',
srcpkg, ubuntu_ver, new_ver) srcpkg, ubuntu_ver, new_ver)
files = dsc_getfiles(dscurl) files = dsc_getfiles(dscname)
source_files = [f for f in files if f.is_source_file()] source_files = [f for f in files if f.is_source_file()]
Logger.debug('Files: %s', str([x.get_name() for x in files])) Logger.debug('Files: %s', str([x.get_name() for x in files]))
Logger.debug('Source files: %s', str([x.get_name() for x in source_files])) Logger.debug('Source files: %s', str([x.get_name() for x in source_files]))
[f.download() for f in files]
if ubuntu_dsc is None: if ubuntu_dsc is None:
ubuntu_files = None ubuntu_files = None
@ -263,7 +228,8 @@ def sync_dsc(dscurl, debian_dist, release, name, email, bugs, keyid=None):
# Do a fake sync if required # Do a fake sync if required
if len(fakesync_files) > 0: if len(fakesync_files) > 0:
# Download Ubuntu files (override Debian source tarballs) # Download Ubuntu files (override Debian source tarballs)
[f.download() for f in fakesync_files] pull_source_pkg('UBUNTU', ubuntu_mirror, ubuntu_source.getComponent(),
srcpkg, ubuntu_ver.full_version)
# change into package directory # change into package directory
directory = srcpkg + '-' + new_ver.upstream_version directory = srcpkg + '-' + new_ver.upstream_version
@ -352,39 +318,52 @@ def sync_dsc(dscurl, debian_dist, release, name, email, bugs, keyid=None):
'Please check build log above.') 'Please check build log above.')
sys.exit(1) sys.exit(1)
def get_debian_dscurl(package, dist, release, version=None, component=None): def fetch_source_pkg(package, dist, version, component, ubuntu_release, mirror):
"""Download the specified source package.
dist, version, component, mirror can all be None.
"""
if dist is None: if dist is None:
dist = "unstable" dist = "unstable"
requested_version = version
if type(version) == str: if type(version) == str:
version = Version(version) version = Version(version)
if version is None or component is None: if version is None or component is None:
debian_srcpkg = getDebianSrcPkg(package, dist) debian_srcpkg = getDebianSrcPkg(package, dist)
try:
src_pkg = getUbuntuSrcPkg(package, release)
ubuntu_version = Version(src_pkg.getVersion())
except udtexceptions.PackageNotFoundException:
ubuntu_version = Version('~')
if ubuntu_version >= Version(debian_srcpkg.getVersion()):
# The LP importer is maybe out of date
debian_srcpkg = requestsync_mail_getDebianSrcPkg(package, dist)
if version is None: if version is None:
version = Version(debian_srcpkg.getVersion()) version = Version(debian_srcpkg.getVersion())
try:
ubuntu_srcpkg = getUbuntuSrcPkg(package, ubuntu_release)
ubuntu_version = Version(ubuntu_srcpkg.getVersion())
except udtexceptions.PackageNotFoundException:
ubuntu_version = Version('~')
if ubuntu_version >= version:
# The LP importer is maybe out of date
debian_srcpkg = requestsync_mail_getDebianSrcPkg(package, dist)
if requested_version is None:
version = Version(debian_srcpkg.getVersion())
if ubuntu_version >= version:
Logger.error("Version in Debian %s (%s) isn't newer than "
"Ubuntu %s (%s)",
version, dist, ubuntu_version, ubuntu_release)
sys.exit(1)
if component is None: if component is None:
component = debian_srcpkg.getComponent() component = debian_srcpkg.getComponent()
assert component in ("main", "contrib", "non-free") assert component in ('main', 'contrib', 'non-free')
if package.startswith("lib"): return pull_source_pkg('DEBIAN', mirror, component, package,
group = package[0:4] version.full_version)
else:
group = package[0]
dsc_file = package + "_" + version.strip_epoch() + ".dsc" def fetch_dsc(dscfile):
dscurl = os.path.join("http://ftp.debian.org/debian/pool", component, group, "Fetch a dsc"
package, dsc_file) url = urlparse.urlparse(dscfile)
return dscurl if not url.scheme:
dscfile = 'file://' + os.path.abspath(dscfile)
cmd = ('dget', '--allow-unauthenticated', '-d', dscfile)
Logger.command(cmd)
subprocess.check_call(cmd)
return os.path.basename(url.path)
def main(): def main():
usage = "%prog [options] <.dsc URL/path or package name>" usage = "%prog [options] <.dsc URL/path or package name>"
@ -424,6 +403,16 @@ def main():
dest="bugs", action="append", default=list(), dest="bugs", action="append", default=list(),
help="Mark Launchpad bug BUG as being fixed by this " help="Mark Launchpad bug BUG as being fixed by this "
"upload.") "upload.")
parser.add_option('-D', '--debian-mirror', metavar='DEBIAN_MIRROR',
dest='debian_mirror',
help='Preferred Debian mirror '
'(default: %s)'
% UDTConfig.defaults['DEBIAN_MIRROR'])
parser.add_option('-U', '--ubuntu-mirror', metavar='UBUNTU_MIRROR',
dest='ubuntu_mirror',
help='Prefeed Ubuntu mirror '
'(default: %s)'
% UDTConfig.defaults['UBUNTU_MIRROR'])
parser.add_option('--no-conf', parser.add_option('--no-conf',
dest='no_conf', default=False, action='store_true', dest='no_conf', default=False, action='store_true',
help="Don't read config files or environment variables.") help="Don't read config files or environment variables.")
@ -441,11 +430,19 @@ def main():
parser.error('Invalid bug number(s) specified: ' parser.error('Invalid bug number(s) specified: '
+ ', '.join(invalid_bug_numbers)) + ', '.join(invalid_bug_numbers))
config = UDTConfig(options.no_conf) if options.component not in (None, "main", "contrib", "non-free"):
parser.error('%s is not a valid Debian component. '
'It should be one of main, contrib, or non-free.'
% options.component)
Logger.verbose = options.verbose
config = UDTConfig(options.no_conf)
if options.debian_mirror is None:
options.debian_mirror = config.get_value('DEBIAN_MIRROR')
if options.ubuntu_mirror is None:
options.ubuntu_mirror = config.get_value('UBUNTU_MIRROR')
if options.uploader_name is None: if options.uploader_name is None:
options.uploader_name = ubu_email(export=False)[0] options.uploader_name = ubu_email(export=False)[0]
if options.uploader_email is None: if options.uploader_email is None:
options.uploader_email = ubu_email(export=False)[1] options.uploader_email = ubu_email(export=False)[1]
@ -454,19 +451,17 @@ def main():
options.release = Launchpad.distributions["ubuntu"].current_series.name options.release = Launchpad.distributions["ubuntu"].current_series.name
if args[0].endswith(".dsc"): if args[0].endswith(".dsc"):
dscurl = args[0] dscfile = args[0]
if '/' in dscfile:
dscfile = fetch_dsc(dscfile)
else: else:
if options.component not in (None, "main", "contrib", "non-free"): dscfile = fetch_source_pkg(args[0], options.dist, options.debversion,
parser.error('%s is not a valid Debian component. ' options.component, options.release,
'It should be one of main, contrib, or non-free.' options.debian_mirror)
% options.component)
dscurl = get_debian_dscurl(args[0], options.dist, options.release,
options.debversion, options.component)
Logger.verbose = options.verbose sync_dsc(dscfile, options.dist, options.release, options.uploader_name,
Logger.debug('.dsc url: %s', dscurl) options.uploader_email, options.bugs, options.ubuntu_mirror,
sync_dsc(dscurl, options.dist, options.release, options.uploader_name, options.keyid)
options.uploader_email, options.bugs, options.keyid)
if __name__ == "__main__": if __name__ == "__main__":
main() main()

View File

@ -2,7 +2,8 @@
# List of classes names for which member attributes should not be checked # List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set). # (useful for classes with attributes dynamically set).
ignored-classes=Launchpad,BaseWrapper,PersonTeam,Distribution,Consumer,Credentials # lpapicache clasess, urlparse
ignored-classes=Launchpad,BaseWrapper,PersonTeam,Distribution,Consumer,Credentials,ParseResult
[FORMAT] [FORMAT]