ubuntu-dev-tools/requestsync

#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# (C) 2007 Canonical Ltd., Steve Kowalik
# Authors:
#  Martin Pitt <martin.pitt@ubuntu.com>
#  Steve Kowalik <stevenk@ubuntu.com>
#  Michael Bienia <geser@ubuntu.com> (python-launchpad-bugs support)
#  Daniel Hahler <ubuntu@thequod.de>
#  Iain Lane <laney@ubuntu.com>
#  Jonathan Davies <jpds@ubuntu.com>
#  Markus Korn <thekorn@gmx.de> (python-launchpadlib support)
#
# ##################################################################
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 2.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# See file /usr/share/common-licenses/GPL-2 for more details.
#
# ##################################################################

import os
import subprocess
import sys
import urllib
import urllib2
from debian_bundle.changelog import Version
from optparse import OptionParser
from time import sleep

# ubuntu-dev-tools modules.
import ubuntutools.lp.cookie as lp_cookie
import ubuntutools.lp.functions as lp_functions
import ubuntutools.lp.libsupport as lp_libsupport
import ubuntutools.lp.urlopener as lp_urlopener
import ubuntutools.lp.udtexceptions as udtexceptions
# https_proxy fix
import ubuntutools.common

launchpad_cookiefile = lp_cookie.prepareLaunchpadCookie()

def checkNeedsSponsorship(srcpkg):
    """
        Check that the user has the appropriate permissions by checking what
        Launchpad returns while authenticating with their cookie.

        If they are an indirect or direct member of the ~ubuntu-dev team on
        Launchpad - sponsorship is not required if the package is in the
        universe / multiverse component.

        If they are in the ~ubuntu-core-dev team, no sponsorship required.

        The prepareLaunchpadCookie function above shall ensure that a cookie
        file exists first.
    """
    if component in ['main', 'restricted']:
        team = "ubuntu-core-dev"
        sponsor = "ubuntu-main-sponsors"
    else:
        team = "motu"
        sponsor = "ubuntu-universe-sponsors"

    if not lp_functions.canUploadPackage(srcpkg):

        # Check if they have a per-package upload permission.
        if lp_functions.isPerPackageUploader(args[0]):
            return "perpackageupload"

        print "You are not a member (direct or indirect) of the '%s' " \
            "team on Launchpad." % team
        print "Your sync request shall require an approval by a member of " \
            "the '%s'\nteam, who shall be subscribed to this bug report." % sponsor
        print "This must be done before it can be processed by a member of " \
            "the Ubuntu Archive team."
        print "If the above is correct please press Enter."
        raw_input_exit_on_ctrlc() # Abort if necessary.
        return True # Sponsorship required.

    # Is a team member, no sponsorship required.
    return False

def checkExistingReports(package):
    """ Check existing bug reports on Launchpad for a possible sync request.

        If found ask for confirmation on filing a request.
    """

    launchpad = None

    try:
        launchpad = lp_libsupport.get_launchpad("ubuntu-dev-tools")
    except ImportError:
        print >> sys.stderr, 'Importing launchpadlib failed. Is ' \
            'python-launchpadlib installed?'
    except IOError, msg:
        # No credentials found.
        print msg

    # Failed to get Launchpad credentials.
    if launchpad is None:
        print >> sys.stderr, "Skipping existing report check, you should "\
            "manually see if there are any at:"
        print "- https://bugs.launchpad.net/ubuntu/+source/%s" % package
        print ""
        return False

    # Fetch the package's bug list from Launchpad.
    pkg = launchpad.distributions["ubuntu"].getSourcePackage(name=package)
    pkgBugList = pkg.searchTasks()

    # Search bug list for other sync requests.
    matchingBugs = [bug for bug in pkgBugList 
                        if "Sync %s" % package in bug.title 
                        or "Please sync %s" % package in bug.title]

    if len(matchingBugs) == 0:
        return # No sync bugs found.

    print "The following bugs could be possible duplicate sync bug(s) on Launchpad:"

    for bug in matchingBugs:
        print " *", bug.title
        print "   -", lp_libsupport.translate_api_web(bug.self_link)
    
    print "Please check the above URLs to verify this before filing a " \
        "possible duplicate report."
    print "Press Ctrl-C to stop filing the bug report now, otherwise " \
         "please press enter."
    raw_input_exit_on_ctrlc()
            
def cur_version_component(sourcepkg, release):
    
    try:
        version = lp_functions.packageVersion(sourcepkg, release)
        component = lp_functions.packageComponent(sourcepkg, release)
        
        return (version, component)
     
    except udtexceptions.PackageNotFoundException:
        
        print "%s doesn't appear to exist in %s, specify -n for a package not in Ubuntu." % (sourcepkg, release)
        sys.exit(1)

def cur_deb_version(sourcepkg, distro):
    '''Return the current debian version of a package in a Debian distro.'''
    madison = subprocess.Popen(['rmadison', '-u', 'debian', '-a', 'source', \
                                '-s', distro, sourcepkg], \
                               stdout=subprocess.PIPE)
    out = madison.communicate()[0]
    assert (madison.returncode == 0)

    try:
        assert out
    except AssertionError:
        print "%s doesn't appear to exist in Debian." % sourcepkg
        sys.exit(1)
    # Work-around for a bug in Debians madison.php script not returning 
    # only the source line 
    for line in out.splitlines():
        if line.find('source') > 0:
            out = line
    
    return out.split('|')[1].rstrip('[]''').strip()

def debian_changelog(sourcepkg, component, version):
    '''Return the Debian changelog from the latest up to the given version
    (exclusive).'''

    base_version = Version(version)

    ch = ''
    subdir = sourcepkg[0]

    if sourcepkg.startswith('lib'):
        subdir = 'lib%s' % sourcepkg[3]

    # Get the debian/changelog file from packages.debian.org.
    try:
        debianChangelogPage = urllib2.urlopen('http://packages.debian.org/changelogs/pool/%s/%s/%s/current/changelog.txt' % (component, subdir, sourcepkg))
    except urllib2.HTTPError, error:
        print >> sys.stderr, "Unable to connect to packages.debian.org. " \
            "Received a %s." % error.code
        sys.exit(1)

    for l in debianChangelogPage:
        if l.startswith(sourcepkg):
            ch_version = l[ l.find("(")+1 : l.find(")") ]
            if Version(ch_version) <= base_version:
                break
        ch += l

    return ch

def debian_component(sourcepkg, distro):
    '''Return the Debian component for the source package.'''
    madison = subprocess.Popen(['rmadison', '-u', 'debian', '-a', 'source', '-s', distro, \
                                sourcepkg], stdout=subprocess.PIPE)
    out = madison.communicate()[0]
    assert (madison.returncode == 0)

    try:
        assert out
    except AssertionError:
        print "%s doesn't appear to exist in Debian." % sourcepkg
        sys.exit(1)
    raw_comp = out.split('|')[2].split('/')
    component = 'main'
    if len(raw_comp) == 2:
        component = raw_comp[1].strip()
    return component

def raw_input_exit_on_ctrlc(*args, **kwargs):
    """A wrapper around raw_input() to exit with a normalized message on Control-C"""
    try:
        return raw_input(*args, **kwargs)
    except KeyboardInterrupt:
        print 'Abort requested. No sync request filed.'
        sys.exit(1)

def get_email_address():
    '''Get the DEBEMAIL environment variable or give an error.'''
    myemailaddr = os.getenv('DEBEMAIL')
    if not myemailaddr:
        print >> sys.stderr, 'The environment variable DEBEMAIL needs to be ' +\
            ' set to make use of this script, unless you use option --lp.'
    return myemailaddr

def mail_bug(source_package, subscribe, status, bugtitle, bugtext, keyid = None):
    '''Submit the sync request per email.
    Return True if email successfully send, otherwise False.'''
    
    import smtplib
    import socket

    to = 'new@bugs.launchpad.net'

    myemailaddr = get_email_address()
    if not myemailaddr:
        return False

    # generate initial mailbody
    mailbody = ''
    if source_package:
        mailbody += ' affects ubuntu/%s\n' % source_package
    else:
        mailbody += ' affects ubuntu\n'
    mailbody = mailbody + ' status %s\n importance wishlist\n subscribe %s\n\n%s' % (status, subscribe, bugtext)

    # prepare sign_command
    sign_command = 'gpg'
    for cmd in ('gpg2', 'gnome-gpg'):
        if os.access('/usr/bin/%s' % cmd, os.X_OK):
            sign_command = cmd

    gpg_command = [sign_command, '--clearsign']
    if keyid:
        gpg_command.extend(('-u', keyid))

    in_confirm_loop = True
    while in_confirm_loop:
        # sign it
        gpg = subprocess.Popen(gpg_command, stdin = subprocess.PIPE, stdout = subprocess.PIPE)
        signed_report = gpg.communicate(mailbody)[0]
        assert gpg.returncode == 0

        # generate email
        mail = '''\
From: %s
To: %s
Subject: %s
Content-Type: text/plain; charset=UTF-8

%s''' % (myemailaddr, to, bugtitle, signed_report)

        # ask for confirmation and allow to edit:
        print mail
        print 'Do you want to edit the report before sending [y/N]? Press Control-C to abort.'
        while 1:
            val = raw_input_exit_on_ctrlc()
            if val.lower() in ('y', 'yes'):
                (bugtitle, mailbody) = edit_report(bugtitle, mailbody)
                break
            elif val.lower() in ('n', 'no', ''):
                in_confirm_loop = False
                break
            else:
                print "Invalid answer"

    # get server address
    mailserver = os.getenv('DEBSMTP')
    if mailserver:
        print 'Using custom SMTP server:', mailserver
    else:
        mailserver = 'fiordland.ubuntu.com'

    # get server port
    mailserver_port = os.getenv('DEBSMTP_PORT')
    if mailserver_port:
        print 'Using custom SMTP port:', mailserver_port
    else:
        mailserver_port = 25

    # connect to the server
    try:
        s = smtplib.SMTP(mailserver, mailserver_port)
    except socket.error, s:
        print >> sys.stderr, "Could not connect to mailserver %s at port %s: %s (%i)" % \
            (mailserver, mailserver_port, s[1], s[0])
        print "The port %s may be firewalled. Please try using requestsync with" \
            % mailserver_port
        print "the '--lp' flag to file a sync request with the launchpadlib " \
            "module."
        return False

    # authenticate to the server
    mailserver_user = os.getenv('DEBSMTP_USER')
    mailserver_pass = os.getenv('DEBSMTP_PASS')
    if mailserver_user and mailserver_pass:
        try:
            s.login(mailserver_user, mailserver_pass)
        except smtplib.SMTPAuthenticationError:
            print 'Error authenticating to the server: invalid username and password.'
            s.quit()
            return False
        except:
            print 'Unknown SMTP error.'
            s.quit()
            return False

    s.sendmail(myemailaddr, to, mail)
    s.quit()
    print 'Sync request mailed.'

    return True

def post_bug(source_package, subscribe, status, bugtitle, bugtext):
    '''Use python-launchpadlib to submit the sync request.
    Return True if successfully posted, otherwise False.'''

    import glob, os.path

    try:
        launchpad = lp_libsupport.get_launchpad("ubuntu-dev-tools")
    except ImportError:
        print >> sys.stderr, 'Importing launchpadlib failed. Is python-launchpadlib installed?'
        return False
    except IOError, msg:
        # No credentials found.
        print msg
        sys.exit(1)

    if source_package:
        product_url = "%subuntu/+source/%s" %(launchpad._root_uri, source_package)
    else:
        # new source package
        product_url = "%subuntu" %launchpad._root_uri
        
    in_confirm_loop = True
    while in_confirm_loop:
        print 'Summary:\n%s\n\nDescription:\n%s' % (bugtitle, bugtext)

        # ask for confirmation and allow to edit:
        print 'Do you want to edit the report before sending [y/N]? Press Control-C to abort.'
        while 1:
            val = raw_input_exit_on_ctrlc()
            if val.lower() in ('y', 'yes'):
                (bugtitle, bugtext) = edit_report(bugtitle, bugtext)
                break
            elif val.lower() in ('n', 'no', ''):
                in_confirm_loop = False
                break
            else:
                print "Invalid answer."

    # Create bug
    bug = launchpad.bugs.createBug(description=bugtext, title=bugtitle, target=product_url)
    
    #newly created bugreports have one task
    task = bug.bug_tasks[0]
    # Only members of ubuntu-bugs can set importance
    if lp_functions.isLPTeamMember('ubuntu-bugs'):
        task.transitionToImportance(importance='Wishlist')
    task.transitionToStatus(status=status)
    
    subscribe_url = "%s~%s" %(launchpad._root_uri, subscribe)
    bug.subscribe(person=subscribe_url)

    print 'Sync request filed as bug #%i: %s' % (bug.id,
        lp_libsupport.translate_api_web(bug.self_link))
    return True

def edit_report(subject, body, changes_required=False):
    """Edit a report (consisting of subject and body) in sensible-editor.
    
    subject and body get decorated, before they are written to the temporary
    file and undecorated after editing again.
    If changes_required is True and the file has not been edited
    (according to its mtime), an error is written to STDERR and the
    program exits.
    Returns (new_subject, new_body).
    """
    import re
    import string

    report = "Summary (one line):\n%s\n\nDescription:\n%s" % (subject, body)

    # Create tempfile and remember mtime
    import tempfile
    report_file = tempfile.NamedTemporaryFile( prefix='requestsync_' )
    report_file.file.write(report)
    report_file.file.flush()
    mtime_before = os.stat( report_file.name ).st_mtime

    # Launch editor
    try:
        editor = subprocess.check_call( ['sensible-editor', report_file.name] )
    except subprocess.CalledProcessError, e:
        print >> sys.stderr, 'Error calling sensible-editor: %s\nAborting.' % (e,)
        sys.exit(1)

    # Check if the tempfile has been changed
    if changes_required:
        report_file_info = os.stat( report_file.name )
        if mtime_before == os.stat( report_file.name ).st_mtime:
            print >> sys.stderr, 'The temporary file %s has not been changed, but you have\nto explain why the Ubuntu changes can be dropped. Aborting. [Press ENTER]' % (report_file.name,)
            raw_input()
            sys.exit(1)

    report_file.file.seek(0)
    report = report_file.file.read()
    report_file.file.close()

    # Undecorate report again:
    (new_subject, new_body) = report.split("\nDescription:\n", 1)
    # Remove prefix and whitespace for subject:
    new_subject = string.rstrip( re.sub("\n", " ", re.sub("^Summary \(one line\):\s*", "", new_subject, 1)) )

    return (new_subject, new_body)


#
# entry point
#

if __name__ == '__main__':
    # Our usage options.
    usage = "Usage: %prog [-d distro] [-k keyid] [-n] [--lp] [-s] [-e] "
    usage += "<source package> <target release> [base version]"
    optParser = OptionParser(usage)

    optParser.add_option("-d", type = "string",
        dest = "dist", default = "unstable",
        help = "Debian distribution to sync from.")
    optParser.add_option("-k", type = "string",
        dest = "keyid", default = None,
        help = "GnuPG key ID to use for signing report.")
    optParser.add_option("-n", action = "store_true",
        dest = "newpkg", default = False,
        help = "Whether package to sync is a new package in Ubuntu.")
    optParser.add_option("--lp", action = "store_true",
        dest = "lpbugs", default = False,
        help = "Specify whether to use the launchpadlib module for filing " \
            "report.")
    optParser.add_option("-s", action = "store_true",
        dest = "sponsor", default = False,
        help = "Force sponsorship requirement (shall be autodetected if not " \
            "specified).")
    optParser.add_option("-e", action = "store_true",
        dest = "ffe", default = False,
        help = "Use this after FeatureFreeze for non-bug fix syncs, changes " \
            "default subscription to the appropriate release team.")

    (options, args) = optParser.parse_args()

    newsource = options.newpkg
    sponsorship = options.sponsor
    keyid = options.keyid
    use_lp_bugs = options.lpbugs
    need_interaction = False
    distro = options.dist
    ffe = options.ffe

    if len(args) not in (2, 3): # no release specified, assume development release
        release = lp_functions.ubuntuDevelopmentSeries()
        print >> sys.stderr, ("Source package / target release missing - assuming %s " %
            release)
    else:
        release = args[1]

    if not use_lp_bugs and not get_email_address():
        sys.exit(1)

    srcpkg = args[0]
    force_base_ver = None

    # Base version specified.
    if len(args) == 3: force_base_ver = args[2]

    (cur_ver, component) = ('0', 'universe') # Let's assume universe
    
    # Find Ubuntu release's package version.
    if not newsource: (cur_ver, component) = cur_version_component(srcpkg, release)

    debiancomponent = debian_component(srcpkg, distro)
    # Find Debian release's package version.
    deb_version = cur_deb_version(srcpkg, distro)

    # Debian and Ubuntu versions are the same - stop.
    if deb_version == cur_ver:
        print 'The versions in Debian and Ubuntu are the same already (%s). Aborting.' % (deb_version,)
        sys.exit(1)

    # -s flag not specified - check if we do need sponsorship.
    if not sponsorship: sponsorship = checkNeedsSponsorship(srcpkg)

    # Check for existing package reports.
    if not newsource and use_lp_bugs: checkExistingReports(srcpkg)

    # Generate bug report.
    subscribe = 'ubuntu-archive'
    status = 'confirmed'
    if sponsorship == True:
        status = 'new'
        if component in ['main', 'restricted']:
            subscribe = 'ubuntu-main-sponsors'
        else:
            subscribe = 'ubuntu-universe-sponsors'
    if ffe == True:
        status = 'new'
        if component in ['main', 'restricted']:
            subscribe = 'ubuntu-release'
        else:
            subscribe = 'motu-release'


    pkg_to_sync = '%s %s (%s) from Debian %s (%s).' % (srcpkg, deb_version, component, distro, debiancomponent)
    title = "Sync %s" % pkg_to_sync
    if ffe == True:
        title = "FFe: " + title
    report = "Please sync %s\n\n" % pkg_to_sync
    
    base_ver = cur_ver
    uidx = base_ver.find('ubuntu')
    if uidx > 0:
        base_ver = base_ver[:uidx]
        need_interaction = True

        print 'Changes have been made to the package in Ubuntu.'
        print 'Please edit the report and give an explanation.'
        print 'Press ENTER to start your editor. Press Control-C to abort now.'
        print 'Not saving the report file will abort the request, too.'
        raw_input_exit_on_ctrlc()
        report += 'Explanation of the Ubuntu delta and why it can be dropped:\n' + \
                  '>>> ENTER_EXPLANATION_HERE <<<\n\n'

    if ffe == True:
        need_interaction = True

        print 'To approve FeatureFreeze exception, you need to state '
        print 'the reason why you feel it is necessary.'
        print 'Press ENTER to start your editor. Press Control-C to abort now.'
        print 'Not saving the report file will abort the request, too.'
        raw_input_exit_on_ctrlc()
        report += 'Explanation of FeatureFreeze exception:\n' + \
                  '>>> ENTER_EXPLANATION_HERE <<<\n\n'


    if sponsorship == 'perpackageupload':
        report += 'Note that I have per-package upload permissions for %s.\n\n' % srcpkg

    uidx = base_ver.find('build')
    if uidx > 0:
        base_ver = base_ver[:uidx]

    if force_base_ver:
        base_ver = force_base_ver

    report += 'Changelog since current %s version %s:\n\n' % (release, cur_ver)
    report += debian_changelog(srcpkg, debiancomponent, base_ver) + '\n'

    if need_interaction:
        (title, report) = edit_report(title, report, changes_required=True)

    # Post sync request using Launchpad interface:
    srcpkg = not newsource and srcpkg or None
    if use_lp_bugs:
        # Map status to the values expected by lp-bugs
        mapping = {'new': 'New', 'confirmed': 'Confirmed'}
        if post_bug(srcpkg, subscribe, mapping[status], title, report):
            sys.exit(0)
        # Abort on error:
        print 'Something went wrong. No sync request filed.'
        sys.exit(1)

    # Mail sync request:
    if mail_bug(srcpkg, subscribe, status, title, report, keyid):
        sys.exit(0)

    print 'Something went wrong. No sync request filed.'
    sys.exit(1)