* requestsync: Extract current Ubuntu delta from changelog entries and provide for editing (LP: #547925)

* Bump python-debian B-D and Depends to 0.1.20 for unicode Changelog reading.

debian/changelog

@@ -7,8 +7,11 @@ ubuntu-dev-tools (0.131) UNRELEASED; urgency=low
     (LP: #844734)
   * Debian source publication records are all Published now, not pending
     (LP: #845487)
-  * requestsync: Add nice error messages to gpg-signing code, rather than
-    simple assertions (LP: #537288)
+  * requestsync:
+    - Add nice error messages to gpg-signing code, rather than simple
+      assertions (LP: #537288)
+    - Extract current Ubuntu delta from changelog entries and provide for
+      editing (LP: #547925)
   * submittodebian:
     - Don't parse the entire changelog, to avoid bumping into past illegal
       version numbers (LP: #727314)
@@ -25,6 +28,8 @@ ubuntu-dev-tools (0.131) UNRELEASED; urgency=low
   * ubuntutools.archive:
     - Add quiet option to silence downloading.
     - Use wget-style progress bar (fixed width) (LP: #845787)
+  * Bump python-debian B-D and Depends to 0.1.20 for unicode Changelog
+    reading.
 
   [ Colin Watson ]
   * syncpackage: Fix typo.

debian/control

@@ -15,7 +15,7 @@ Build-Depends: dctrl-tools,
                pylint,
                python-all (>= 2.6.5-13~),
                python-apt (>= 0.7.93~),
-               python-debian (>= 0.1.15),
+               python-debian (>= 0.1.20~),
                python-gnupginterface,
                python-launchpadlib (>= 1.5.7),
                python-mox,
@@ -36,7 +36,7 @@ Depends: binutils,
          dpkg-dev,
          lsb-release,
          python-apt (>= 0.7.93~),
-         python-debian (>= 0.1.15),
+         python-debian (>= 0.1.20~),
          python-launchpadlib (>= 1.5.7),
          python-lazr.restfulclient,
          sudo,

requestsync

@@ -139,6 +139,7 @@ def main():
         from ubuntutools.requestsync.lp import (checkExistingReports,
                                                 getDebianSrcPkg,
                                                 getUbuntuSrcPkg,
+                                                getUbuntuDeltaChangelog,
                                                 needSponsorship, postBug)
         from ubuntutools.lp.lpapicache import Distribution, Launchpad
         # See if we have LP credentials and exit if we don't -
@@ -152,6 +153,7 @@ def main():
         from ubuntutools.requestsync.mail import (checkExistingReports,
                                                   getDebianSrcPkg,
                                                   getUbuntuSrcPkg,
+                                                  getUbuntuDeltaChangelog,
                                                   mailBug, needSponsorship)
         if not any(x in os.environ for x in ('UBUMAIL', 'DEBEMAIL', 'EMAIL')):
             print >> sys.stderr, (
@@ -262,8 +264,9 @@ def main():
         print ('Changes have been made to the package in Ubuntu.\n'
                'Please edit the report and give an explanation.\n'
                'Not saving the report file will abort the request.')
-        report += ('Explanation of the Ubuntu delta and why it can be '
-                   'dropped:\n>>> ENTER_EXPLANATION_HERE <<<\n\n')
+        report += (u'Explanation of the Ubuntu delta and why it can be '
+                   u'dropped:\n%s\n>>> ENTER_EXPLANATION_HERE <<<\n\n'
+                   % getUbuntuDeltaChangelog(ubuntu_srcpkg))
 
     if ffe:
         need_interaction = True

ubuntutools/requestsync/common.py

@@ -38,11 +38,10 @@ def raw_input_exit_on_ctrlc(*args, **kwargs):
         print '\nAbort requested. No sync request filed.'
         sys.exit(1)
 
-# TODO: Move this into requestsync.mail, and implement an LP version
-# when LP: #833384 is fixed
-def getDebianChangelog(srcpkg, version):
+def getChangelog(srcpkg, distro):
     '''
-    Return the new changelog entries since 'version'.
+    Download and return a parsed changelog for srcpackage, from
+    packages.debian.org or changelogs.ubuntu.com
     '''
     pkgname = srcpkg.getPackageName()
     pkgversion = srcpkg.getVersion()
@@ -54,26 +53,36 @@ def getDebianChangelog(srcpkg, version):
     # Strip epoch from version
     if ':' in pkgversion:
         pkgversion = pkgversion[pkgversion.find(':')+1:]
+    extension = ''
+    if distro == 'debian':
+        base = 'http://packages.debian.org/'
+        extension = '.txt'
+    elif distro == 'ubuntu':
+        base = 'http://changelogs.ubuntu.com/'
 
-    # Get the debian changelog file from packages.debian.org
+    url = os.path.join(base, 'changelogs', 'pool', component, subdir, pkgname,
+                       pkgname + '_' + pkgversion, 'changelog' + extension)
     try:
-        changelog = urllib2.urlopen('http://packages.debian.org/changelogs/pool'
-                                    '/%s/%s/%s/%s_%s/changelog.txt'
-                                    % (component, subdir, pkgname, pkgname,
-                                       pkgversion))
+        return Changelog(urllib2.urlopen(url))
     except urllib2.HTTPError, error:
-        print >> sys.stderr, ('Unable to connect to packages.debian.org: %s'
-                              % error)
+        print >> sys.stderr, ('Unable to connect to %s: %s' % (base, error))
         return None
 
-    new_entries = ''
-    changelog = Changelog(changelog.read())
+# TODO: Move this into requestsync.mail, and implement an LP version
+# when LP: #833384 is fixed
+def getDebianChangelog(srcpkg, version):
+    '''
+    Return the new changelog entries since 'version'.
+    '''
+    changelog = getChangelog(srcpkg, 'debian')
+    if changelog is None:
+        return None
+    new_entries = []
     for block in changelog:
-        if block.version > version:
-            # see also Debian #561805
-            new_entries += unicode(str(block).decode('utf-8'))
-
-    return new_entries
+        if block.version <= version:
+            break
+        new_entries.append(unicode(block))
+    return u''.join(new_entries)
 
 def edit_report(subject, body, changes_required = False):
     '''

ubuntutools/requestsync/lp.py

@@ -20,6 +20,10 @@
 #   Please see the /usr/share/common-licenses/GPL-2 file for the full text
 #   of the GNU General Public License license.
 
+import re
+import urllib2
+
+from debian.deb822 import Changes
 from distro_info import DebianDistroInfo
 
 from ubuntutools.requestsync.common import raw_input_exit_on_ctrlc
@@ -68,7 +72,7 @@ def checkExistingReports(srcpkg):
     '''
 
     # Fetch the package's bug list from Launchpad
-    pkg = Distribution('ubuntu').getSourcePackage(name = srcpkg)
+    pkg = Distribution('ubuntu').getSourcePackage(name=srcpkg)
     pkgBugList = pkg.searchTasks(status=["Incomplete", "New", "Confirmed",
                                          "Triaged", "In Progress", "Fix Committed"],
                                  omit_duplicates=True)
@@ -86,6 +90,40 @@ def checkExistingReports(srcpkg):
             raw_input_exit_on_ctrlc('Press [Enter] to continue or [Ctrl-C] '
                                     'to abort. ')
 
+def getUbuntuDeltaChangelog(srcpkg):
+    '''
+    Download the Ubuntu changelog and extract the entries since the last sync
+    from Debian.
+    '''
+    archive = Distribution('ubuntu').getArchive()
+    spph = archive.getPublishedSources(source_name=srcpkg.getPackageName(),
+                                       exact_match=True, pocket='Release')
+    debian_info = DebianDistroInfo()
+    topline = re.compile(r'^(\w%(name_chars)s*) \(([^\(\) \t]+)\)'
+                         r'((\s+%(name_chars)s+)+)\;'
+                         % {'name_chars': '[-+0-9a-z.]'},
+                         re.IGNORECASE)
+    delta = []
+    for record in spph:
+        changes_url = record.changesFileUrl()
+        if changes_url is None:
+            continue
+        changes = Changes(urllib2.urlopen(changes_url))
+        for line in changes['Changes'].splitlines():
+            line = line[1:]
+            m = topline.match(line)
+            if m:
+                distribution = m.group(3).split()[0].split('-')[0]
+                if debian_info.valid(distribution):
+                    break
+            if line.startswith(u'  '):
+                delta.append(line)
+        else:
+            continue
+        break
+
+    return '\n'.join(delta)
+
 def postBug(srcpkg, subscribe, status, bugtitle, bugtext):
     '''
     Use the LP API to file the sync request.

ubuntutools/requestsync/mail.py

@@ -30,7 +30,8 @@ from devscripts.logger import Logger
 from distro_info import DebianDistroInfo
 
 from ubuntutools.archive import rmadison, FakeSPPH
-from ubuntutools.requestsync.common import raw_input_exit_on_ctrlc
+from ubuntutools.requestsync.common import (getChangelog,
+                                            raw_input_exit_on_ctrlc)
 from ubuntutools import subprocess
 from ubuntutools.lp.udtexceptions import PackageNotFoundException
 
@@ -39,6 +40,7 @@ __all__ = [
     'getUbuntuSrcPkg',
     'needSponsorship',
     'checkExistingReports',
+    'getUbuntuDeltaChangelog',
     'mailBug',
 ]
 
@@ -90,6 +92,25 @@ def checkExistingReports(srcpkg):
            'for duplicate sync requests before continuing.' % srcpkg)
     raw_input_exit_on_ctrlc('Press [Enter] to continue or [Ctrl-C] to abort. ')
 
+def getUbuntuDeltaChangelog(srcpkg):
+    '''
+    Download the Ubuntu changelog and extract the entries since the last sync
+    from Debian.
+    '''
+    changelog = getChangelog(srcpkg, 'ubuntu')
+    if changelog is None:
+        return u''
+    delta = []
+    debian_info = DebianDistroInfo()
+    for block in changelog:
+        distribution = block.distributions.split()[0].split('-')[0]
+        if debian_info.valid(distribution):
+            break
+        delta += [unicode(change) for change in block.changes()
+                  if change.strip()]
+
+    return u'\n'.join(delta)
+
 def mailBug(srcpkg, subscribe, status, bugtitle, bugtext, bug_mail_domain,
             keyid, myemailaddr, mailserver_host, mailserver_port,
             mailserver_user, mailserver_pass):