From 907061c15ec3b41587fbb613221a0cd1631a589c Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 22 Jan 2020 14:09:58 -0500 Subject: [PATCH] lpapicache: support 'include_meta' param in binaryFileUrls() support the param for both native BPPH as well as Debian Snapshot emulated BPPH add sourceFileUrls() function add helper functions to get file urls, checksums, and size --- ubuntutools/archive.py | 59 ++++++++++++- ubuntutools/lp/lpapicache.py | 165 +++++++++++++++++++++++++++++++++-- 2 files changed, 215 insertions(+), 9 deletions(-) diff --git a/ubuntutools/archive.py b/ubuntutools/archive.py index e8c7f5b..9d1d3d8 100644 --- a/ubuntutools/archive.py +++ b/ubuntutools/archive.py @@ -1094,7 +1094,7 @@ class SnapshotFile(object): @property def size(self): - return self._obj['size'] + return int(self._obj['size']) @property def date(self): @@ -1188,6 +1188,37 @@ class SnapshotSPPH(object): def getComponent(self): return self._pkg.component + def sourceFileUrls(self, include_meta=False): + if include_meta: + return [{'url': f.getUrl(), + 'filename': f.name, + 'sha1': f.getHash(), + 'sha256': None, + 'size': f.size} + for f in self._pkg.getFiles()] + return [f.getUrl() for f in self._pkg.getFiles()] + + def sourceFileUrl(self, filename): + for f in self.sourceFileUrls(include_meta=True): + if filename == f['filename']: + return f['url'] + return None + + def sourceFileSha1(self, url_or_filename): + for f in self.sourceFileUrls(include_meta=True): + if url_or_filename in [f['url'], f['filename']]: + return f['sha1'] + return None + + def sourceFileSha256(self, url_or_filename): + return None + + def sourceFileSize(self, url_or_filename): + for f in self.sourceFileUrls(include_meta=True): + if url_or_filename in [f['url'], f['filename']]: + return int(f['size']) + return 0 + def getChangelog(self, since_version=None): ''' Return the changelog, optionally since a particular version @@ -1282,9 +1313,33 @@ class SnapshotBPPH(object): def getComponent(self): return self._file.component - def binaryFileUrls(self): + def binaryFileUrls(self, include_meta=False): + if include_meta: + return [{'url': self.getUrl(), + 'filename': self.getFileName(), + 'sha1': self._file.getHash(), + 'sha256': None, + 'size': self._file.size}] return [self.getUrl()] + def binaryFileUrl(self, filename): + if filename == self.getFileName(): + return self.getUrl() + return None + + def binaryFileSha1(self, url_or_filename): + if url_or_filename in [self.getUrl(), self.getFileName()]: + return self._file.getHash() + return None + + def binaryFileSha256(self, url_or_filename): + return None + + def binaryFileSize(self, url_or_filename): + if url_or_filename in [self.getUrl(), self.getFileName()]: + return int(self._file.size) + return 0 + def getBuild(self): return None diff --git a/ubuntutools/lp/lpapicache.py b/ubuntutools/lp/lpapicache.py index 28030e7..ca607c7 100644 --- a/ubuntutools/lp/lpapicache.py +++ b/ubuntutools/lp/lpapicache.py @@ -26,6 +26,7 @@ # httplib2.debuglevel = 1 import collections +import os import re from debian.changelog import Changelog @@ -33,6 +34,7 @@ from httplib2 import Http, HttpLib2Error from launchpadlib.launchpad import Launchpad as LP from launchpadlib.errors import HTTPError from lazr.restfulclient.resource import Entry +from urllib.parse import urlparse from ubuntutools.version import Version from ubuntutools.lp import (service, api_version) @@ -700,6 +702,7 @@ class SourcePackagePublishingHistory(BaseWrapper): self._changelog = None self._binaries = {} self._distro_series = None + self._source_urls = None # Don't share _builds between different # SourcePackagePublishingHistory objects if '_builds' not in self.__dict__: @@ -794,6 +797,84 @@ class SourcePackagePublishingHistory(BaseWrapper): new_entries.append(str(block)) return ''.join(new_entries) + def sourceFileUrls(self, include_meta=False): + ''' + Return the URL for this source publication's files. + + The include_meta param changes the return value; + when it is False (the default), an array of url strings is + returned. When include_meta is True, an array is returned + with dicts, containing the entries: + url: the url string + sha1: the SHA1 checksum of the source file (if provided) + sha256: the SHA256 checksum of the source file + size: the size of the source file + Also, this function adds a 'filename' field: + filename: the filename parsed from the url path + ''' + if not self._source_urls: + urls = self._lpobject.sourceFileUrls(include_meta=True) + if not urls: + Logger.warning('SPPH %s_%s has no sourceFileUrls' % + (self.getPackageName(), self.getVersion())) + for u in urls: + u['filename'] = os.path.basename(urlparse(u['url']).path) + self._source_urls = urls + + if include_meta: + return list(self._source_urls) + return [f['url'] for f in self._source_urls] + + def sourceFileUrl(self, filename): + ''' + Returns the URL for the specified source filename. + + If the filename is not found in the sourceFileUrls(), this returns None. + ''' + for f in self.sourceFileUrls(include_meta=True): + if filename == f['filename']: + return f['url'] + return None + + def sourceFileSha1(self, url_or_filename): + ''' + Returns the SHA1 checksum for the specified source file url. + + If the url is not found in the sourceFileUrls(), this returns None. + + The url may be specified as a filename. + ''' + for f in self.sourceFileUrls(include_meta=True): + if url_or_filename in [f['url'], f['filename']]: + return f['sha1'] + return None + + def sourceFileSha256(self, url_or_filename): + ''' + Returns the SHA256 checksum for the specified source file url. + + If the url is not found in the sourceFileUrls(), this returns None. + + The url may be specified as a filename. + ''' + for f in self.sourceFileUrls(include_meta=True): + if url_or_filename in [f['url'], f['filename']]: + return f['sha256'] + return None + + def sourceFileSize(self, url_or_filename): + ''' + Returns the size for the specified source file url. + + If the url is not found in the sourceFileUrls(), this returns 0. + + The url may be specified as a filename. + ''' + for f in self.sourceFileUrls(include_meta=True): + if url_or_filename in [f['url'], f['filename']]: + return int(f['size']) + return 0 + def getBinaries(self, arch=None, name=None, ext=None): ''' Returns the resulting BinaryPackagePublishingHistorys. @@ -936,6 +1017,7 @@ class BinaryPackagePublishingHistory(BaseWrapper): def __init__(self, *args): self._arch = None self._ext = None + self._binary_urls = None @property def arch(self): @@ -968,20 +1050,89 @@ class BinaryPackagePublishingHistory(BaseWrapper): ''' return self._lpobject.component_name - def binaryFileUrls(self): + def binaryFileUrls(self, include_meta=False): ''' Return the URL for this binary publication's files. Only available in the devel API, not 1.0 + + The include_meta param changes the return value; + when it is False (the default), an array of url strings is + returned (but typically there is only a single url in the array). + When include_meta is True, an array (again, with typically only one + entry) is returned with dicts, containing the entries: + url: the url string + sha1: the SHA1 checksum of the binary file + sha256: the SHA256 checksum of the binary file + size: the size of the binary file + Also, this function adds a 'filename' field: + filename: the filename parsed from the url path ''' - try: - urls = self._lpobject.binaryFileUrls() + if not self._binary_urls: + try: + urls = self._lpobject.binaryFileUrls(include_meta=True) + except AttributeError: + raise AttributeError("binaryFileUrls can only be found in lpapi " + "devel, not 1.0. Login using devel to have it.") if not urls: Logger.warning('BPPH %s_%s has no binaryFileUrls' % (self.getPackageName(), self.getVersion())) - return urls - except AttributeError: - raise AttributeError("binaryFileUrls can only be found in lpapi " - "devel, not 1.0. Login using devel to have it.") + for u in urls: + u['filename'] = os.path.basename(urlparse(u['url']).path) + self._binary_urls = urls + + if include_meta: + return list(self._binary_urls) + return [f['url'] for f in self._binary_urls] + + def binaryFileUrl(self, filename): + ''' + Returns the URL for the specified binary filename. + + If the filename is not found in the binaryFileUrls(), this returns None. + ''' + for f in self.binaryFileUrls(include_meta=True): + if filename == f['filename']: + return f['url'] + return None + + def binaryFileSha1(self, url_or_filename): + ''' + Returns the SHA1 checksum for the specified binary file url. + + If the url is not found in the binaryFileUrls(), this returns None. + + The url may be specified as a filename. + ''' + for f in self.binaryFileUrls(include_meta=True): + if url_or_filename in [f['url'], f['filename']]: + return f['sha1'] + return None + + def binaryFileSha256(self, url_or_filename): + ''' + Returns the SHA256 checksum for the specified binary file url. + + If the url is not found in the binaryFileUrls(), this returns None. + + The url may be specified as a filename. + ''' + for f in self.binaryFileUrls(include_meta=True): + if url_or_filename in [f['url'], f['filename']]: + return f['sha256'] + return None + + def binaryFileSize(self, url_or_filename): + ''' + Returns the size for the specified binary file url. + + If the url is not found in the binaryFileUrls(), this returns 0. + + The url may be specified as a filename. + ''' + for f in self.binaryFileUrls(include_meta=True): + if url_or_filename in [f['url'], f['filename']]: + return int(f['size']) + return 0 def getBuild(self): '''