Add changelog retrieval to lpapicache, and use this in syncpackage and

requestsync. The changelogs should be available in Launchpad sooner than
Debian PTS.

debian/changelog

@@ -18,6 +18,9 @@ ubuntu-dev-tools (0.135) UNRELEASED; urgency=low
       testing checklist.
   * Don't allow boilerplate prompts through in submittodebian and requestsync
     (LP: #887336)
+  * Add changelog retrieval to lpapicache, and use this in syncpackage and
+    requestsync. The changelogs should be available in Launchpad sooner than
+    Debian PTS.
 
  -- Stefano Rivera <stefanor@debian.org>  Sat, 12 Nov 2011 13:09:05 +0200
 

debian/copyright

@@ -14,7 +14,6 @@ Files: *
        requestsync
        setup.py
        ubuntu-iso
-       ubuntutools/requestsync/common.py
        ubuntutools/requestsync/lp.py
        ubuntutools/requestsync/mail.py
 Copyright: 2007,      Albert Damen <albrt@gmx.net>

requestsync

@@ -37,7 +37,6 @@ from ubuntutools.config import UDTConfig, ubu_email
 from ubuntutools.lp import udtexceptions
 from ubuntutools.misc import require_utf8
 from ubuntutools.question import confirmation_prompt, EditBugReport
-from ubuntutools.requestsync.common import get_debian_changelog
 
 #
 # entry point
@@ -158,7 +157,8 @@ def main():
         # cannot continue in this case
 
         try:
-            Launchpad.login(service=options.lpinstance)
+            # devel for changelogUrl()
+            Launchpad.login(service=options.lpinstance, api_version='devel')
         except IOError:
             sys.exit(1)
     else:
@@ -298,7 +298,7 @@ def main():
     else:
         report += ('Changelog entries since current %s version %s:\n\n'
                    % (release, ubuntu_version))
-    changelog = get_debian_changelog(debian_srcpkg, base_version)
+    changelog = debian_srcpkg.getChangelog(since_version=base_version)
     if not changelog:
         if not options.missing_changelog_ok:
             print >> sys.stderr, ("E: Did not retrieve any changelog entries. "

syncpackage

@@ -37,7 +37,6 @@ from lazr.restfulclient.errors import HTTPError
 from ubuntutools.archive import (DebianSourcePackage, UbuntuSourcePackage,
                                  DownloadError)
 from ubuntutools.config import UDTConfig, ubu_email
-from ubuntutools.requestsync.common import get_debian_changelog
 from ubuntutools.requestsync.mail import (get_debian_srcpkg
                                           as requestsync_mail_get_debian_srcpkg)
 from ubuntutools.requestsync.lp import get_debian_srcpkg, get_ubuntu_srcpkg
@@ -375,7 +374,7 @@ def copy(src_pkg, release, bugs, simulate=False, force=False):
                       src_pkg.source, ubuntu_series, ubuntu_pocket,
                       src_pkg.version)
 
-    changes = get_debian_changelog(debian_spph, base_version)
+    changes = debian_spph.getChangelog(since_version=base_version)
     if changes:
         changes = changes.strip()
         Logger.normal("New changes:\n%s", changes)
@@ -603,6 +602,7 @@ def main():
     if options.lpinstance is None:
         options.lpinstance = config.get_value('LPINSTANCE')
     try:
+        # devel for copyPackage and changelogUrl
         Launchpad.login(service=options.lpinstance, api_version='devel')
     except IOError:
         sys.exit(1)

ubuntutools/archive.py

@@ -36,6 +36,7 @@ import urlparse
 import re
 import sys
 
+from debian.changelog import Changelog, Version
 import debian.deb822
 import debian.debian_support
 
@@ -443,11 +444,12 @@ class DebianSourcePackage(SourcePackage):
                 comp = record['component']
                 if record['version'] == self.version.full_version:
                     self._spph = FakeSPPH(record['source'], record['version'],
-                                          comp)
+                                          comp, 'debian')
                     return self._spph
 
             Logger.normal('Guessing component from most recent upload')
-            self._spph = FakeSPPH(self.source, self.version.full_version, comp)
+            self._spph = FakeSPPH(self.source, self.version.full_version, comp,
+                                  'debian')
         return self._spph
 
     def _source_urls(self, name):
@@ -527,10 +529,12 @@ class FakeSPPH(object):
     """Provide the same interface as
     ubuntutools.lpapicache.SourcePackagePublishingHistory
     """
-    def __init__(self, name, version, component):
+    def __init__(self, name, version, component, distribution):
         self.name = name
         self.version = version
         self.component = component
+        self.distribution = distribution
+        self._changelog = None
 
     def getPackageName(self):
         return self.name
@@ -541,6 +545,48 @@ class FakeSPPH(object):
     def getComponent(self):
         return self.component
 
+    def getChangelog(self, since_version=None):
+        '''
+        Return the changelog, optionally since a particular version
+        May return None if the changelog isn't available
+        '''
+        if self._changelog is None:
+            if self.name.startswith('lib'):
+                subdir = 'lib%s' % self.name[3]
+            else:
+                subdir = self.name[0]
+            # Strip epoch from version
+            pkgversion = self.version.split(':', 1)[-1]
+            extension = ''
+            if self.distribution == 'debian':
+                base = 'http://packages.debian.org/'
+                extension = '.txt'
+            elif self.distribution == 'ubuntu':
+                base = 'http://changelogs.ubuntu.com/'
+
+            url = os.path.join(base, 'changelogs', 'pool',
+                               self.component, subdir, self.name,
+                               self.name + '_' + pkgversion,
+                               'changelog' + extension)
+            try:
+                self._changelog = urllib2.urlopen(url).read()
+            except urllib2.HTTPError, error:
+                print >> sys.stderr, ('%s: %s' % (url, error))
+                return None
+
+        if since_version is None:
+            return self._changelog
+
+        if isinstance(since_version, basestring):
+            since_version = Version(since_version)
+
+        new_entries = []
+        for block in Changelog(self._changelog):
+            if block.version <= since_version:
+                break
+            new_entries.append(unicode(block))
+        return u''.join(new_entries)
+
 
 def rmadison(url, package, suite=None, arch=None):
     "Call rmadison and parse the result"

ubuntutools/lp/lpapicache.py

@@ -25,7 +25,9 @@
 #httplib2.debuglevel = 1
 
 import sys
+import urllib2
 
+from debian.changelog import Changelog, Version
 from launchpadlib.launchpad import Launchpad as LP
 from launchpadlib.errors import HTTPError
 from lazr.restfulclient.resource import Entry
@@ -349,6 +351,7 @@ class SourcePackagePublishingHistory(BaseWrapper):
     resource_type = 'source_package_publishing_history'
 
     def __init__(self, *args):
+        self._changelog = None
         # Don't share _builds between different
         # SourcePackagePublishingHistory objects
         if '_builds' not in self.__dict__:
@@ -372,6 +375,39 @@ class SourcePackagePublishingHistory(BaseWrapper):
         '''
         return self._lpobject.component_name
 
+    def getChangelog(self, since_version=None):
+        '''
+        Return the changelog, optionally since a particular version
+        May return None if the changelog isn't available
+        Only available in the devel API, not 1.0
+        '''
+        if self._changelog is None:
+            url = self._lpobject.changelogUrl()
+            if url is None:
+                print >> sys.stderr, ('E: No changelog available for %s %s',
+                                      (self.getPackageName(),
+                                       self.getVersion()))
+                return None
+
+            try:
+                self._changelog = urllib2.urlopen(url).read()
+            except urllib2.HTTPError, error:
+                print >> sys.stderr, ('%s: %s' % (url, error))
+                return None
+
+        if since_version is None:
+            return self._changelog
+
+        if isinstance(since_version, basestring):
+            since_version = Version(since_version)
+
+        new_entries = []
+        for block in Changelog(self._changelog):
+            if block.version <= since_version:
+                break
+            new_entries.append(unicode(block))
+        return u''.join(new_entries)
+
     def _fetch_builds(self):
         '''Populate self._builds with the build records.'''
         builds = self.getBuilds()

ubuntutools/requestsync/common.py

@@ -1,73 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-#   common.py - common methods used by requestsync
-#
-#   Copyright © 2009 Michael Bienia <geser@ubuntu.com>
-#
-#   This module may contain code written by other authors/contributors to
-#   the main requestsync script. See there for their names.
-#
-#   This program is free software; you can redistribute it and/or
-#   modify it under the terms of the GNU General Public License
-#   as published by the Free Software Foundation; version 2
-#
-#   This program is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#   GNU General Public License for more details.
-#
-#   Please see the /usr/share/common-licenses/GPL-2 file for the full text
-#   of the GNU General Public License license.
-
-import os
-import sys
-import urllib2
-from debian.changelog import Changelog
-
-from ubuntutools import subprocess
-
-def get_changelog(srcpkg, distro):
-    '''
-    Download and return a parsed changelog for srcpackage, from
-    packages.debian.org or changelogs.ubuntu.com
-    '''
-    pkgname = srcpkg.getPackageName()
-    pkgversion = srcpkg.getVersion()
-    component = srcpkg.getComponent()
-    if pkgname.startswith('lib'):
-        subdir = 'lib%s' % pkgname[3]
-    else:
-        subdir = pkgname[0]
-    # Strip epoch from version
-    if ':' in pkgversion:
-        pkgversion = pkgversion[pkgversion.find(':')+1:]
-    extension = ''
-    if distro == 'debian':
-        base = 'http://packages.debian.org/'
-        extension = '.txt'
-    elif distro == 'ubuntu':
-        base = 'http://changelogs.ubuntu.com/'
-
-    url = os.path.join(base, 'changelogs', 'pool', component, subdir, pkgname,
-                       pkgname + '_' + pkgversion, 'changelog' + extension)
-    try:
-        return Changelog(urllib2.urlopen(url))
-    except urllib2.HTTPError, error:
-        print >> sys.stderr, ('%s: %s' % (url, error))
-        return None
-
-# TODO: Move this into requestsync.mail, and implement an LP version
-# when LP: #833384 is fixed
-def get_debian_changelog(srcpkg, version):
-    '''
-    Return the new changelog entries since 'version'.
-    '''
-    changelog = get_changelog(srcpkg, 'debian')
-    if changelog is None:
-        return None
-    new_entries = []
-    for block in changelog:
-        if block.version <= version:
-            break
-        new_entries.append(unicode(block))
-    return u''.join(new_entries)

ubuntutools/requestsync/mail.py

@@ -25,12 +25,11 @@ import sys
 import smtplib
 import socket
 
-from debian.changelog import Version
+from debian.changelog import Changelog, Version
 from devscripts.logger import Logger
 from distro_info import DebianDistroInfo
 
 from ubuntutools.archive import rmadison, FakeSPPH
-from ubuntutools.requestsync.common import get_changelog
 from ubuntutools.question import confirmation_prompt, YesNoQuestion
 from ubuntutools import subprocess
 from ubuntutools.lp.udtexceptions import PackageNotFoundException
@@ -57,7 +56,7 @@ def _get_srcpkg(distro, name, release):
                                        % (name, distro.capitalize(), release))
     pkg = max(lines, key=lambda x: Version(x['version']))
 
-    return FakeSPPH(pkg['source'], pkg['version'], pkg['component'])
+    return FakeSPPH(pkg['source'], pkg['version'], pkg['component'], distro)
 
 def get_debian_srcpkg(name, release):
     return _get_srcpkg('debian', name, release)
@@ -92,7 +91,7 @@ def get_ubuntu_delta_changelog(srcpkg):
     Download the Ubuntu changelog and extract the entries since the last sync
     from Debian.
     '''
-    changelog = get_changelog(srcpkg, 'ubuntu')
+    changelog = Changelog(srcpkg.getChangelog())
     if changelog is None:
         return u''
     delta = []