kumquat-buildroot/support/scripts/scanpypi
Yegor Yefremov 8a64ade2f3 scanpypi: rework runtime dependency handling
In most cases Python's package dependencies found in setup.py are
runtime dependencies and hence don't need to be mentioned in *.mk
file.

Also add '# runtime' tag to select statements in Config.in.

__create_mk_requirements() itself is left for future uses (cffi backend
handling etc.).

Signed-off-by: Yegor Yefremov <yegorslists@googlemail.com>
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
2016-07-01 00:03:54 +02:00

649 lines
25 KiB
Python
Executable File

#!/usr/bin/python2
"""
Utility for building Buildroot packages for existing PyPI packages
Any package built by scanpypi should be manually checked for
errors.
"""
from __future__ import print_function
import argparse
import json
import urllib2
import sys
import os
import shutil
import StringIO
import tarfile
import zipfile
import errno
import hashlib
import re
import textwrap
import tempfile
import imp
from functools import wraps
def setup_decorator(func, method):
"""
Decorator for distutils.core.setup and setuptools.setup.
Puts the arguments with which setup is called as a dict
Add key 'method' which should be either 'setuptools' or 'distutils'.
Keyword arguments:
func -- either setuptools.setup or distutils.core.setup
method -- either 'setuptools' or 'distutils'
"""
@wraps(func)
def closure(*args, **kwargs):
# Any python packages calls its setup function to be installed.
# Argument 'name' of this setup function is the package's name
BuildrootPackage.setup_args[kwargs['name']] = kwargs
BuildrootPackage.setup_args[kwargs['name']]['method'] = method
return closure
# monkey patch
import setuptools
setuptools.setup = setup_decorator(setuptools.setup, 'setuptools')
import distutils
distutils.core.setup = setup_decorator(setuptools.setup, 'distutils')
def find_file_upper_case(filenames, path='./'):
"""
List generator:
Recursively find files that matches one of the specified filenames.
Returns a relative path starting with path argument.
Keyword arguments:
filenames -- List of filenames to be found
path -- Path to the directory to search
"""
for root, dirs, files in os.walk(path):
for file in files:
if file.upper() in filenames:
yield (os.path.join(root, file))
def pkg_buildroot_name(pkg_name):
"""
Returns the Buildroot package name for the PyPI package pkg_name.
Remove all non alphanumeric characters except -
Also lowers the name and adds 'python-' suffix
Keyword arguments:
pkg_name -- String to rename
"""
name = re.sub('[^\w-]', '', pkg_name.lower())
prefix = 'python-'
pattern = re.compile('^(?!' + prefix + ')(.+?)$')
name = pattern.sub(r'python-\1', name)
return name
class DownloadFailed(Exception):
pass
class BuildrootPackage():
"""This class's methods are not meant to be used individually please
use them in the correct order:
__init__
download_package
extract_package
load_module
get_requirements
create_package_mk
create_hash_file
create_config_in
"""
setup_args = {}
def __init__(self, real_name, pkg_folder):
self.real_name = real_name
self.buildroot_name = pkg_buildroot_name(self.real_name)
self.pkg_dir = os.path.join(pkg_folder, self.buildroot_name)
self.mk_name = self.buildroot_name.upper().replace('-', '_')
self.as_string = None
self.md5_sum = None
self.metadata = None
self.metadata_name = None
self.metadata_url = None
self.pkg_req = None
self.setup_metadata = None
self.tmp_extract = None
self.used_url = None
self.filename = None
self.url = None
self.version = None
def fetch_package_info(self):
"""
Fetch a package's metadata from the python package index
"""
self.metadata_url = 'https://pypi.python.org/pypi/{pkg}/json'.format(
pkg=self.real_name)
try:
pkg_json = urllib2.urlopen(self.metadata_url).read().decode()
except urllib2.HTTPError as error:
print('ERROR:', error.getcode(), error.msg, file=sys.stderr)
print('ERROR: Could not find package {pkg}.\n'
'Check syntax inside the python package index:\n'
'https://pypi.python.org/pypi/ '
.format(pkg=self.real_name))
raise
except urllib2.URLError:
print('ERROR: Could not find package {pkg}.\n'
'Check syntax inside the python package index:\n'
'https://pypi.python.org/pypi/ '
.format(pkg=self.real_name))
raise
self.metadata = json.loads(pkg_json)
self.version = self.metadata['info']['version']
self.metadata_name = self.metadata['info']['name']
def download_package(self):
"""
Download a package using metadata from pypi
"""
try:
self.metadata['urls'][0]['filename']
except IndexError:
print(
'Non-conventional package, ',
'please check carefully after creation')
self.metadata['urls'] = [{
'packagetype': 'sdist',
'url': self.metadata['info']['download_url'],
'md5_digest': None}]
# In this case, we can't get the name of the downloaded file
# from the pypi api, so we need to find it, this should work
urlpath = urllib2.urlparse.urlparse(
self.metadata['info']['download_url']).path
# urlparse().path give something like
# /path/to/file-version.tar.gz
# We use basename to remove /path/to
self.metadata['urls'][0]['filename'] = os.path.basename(urlpath)
for download_url in self.metadata['urls']:
if 'bdist' in download_url['packagetype']:
continue
try:
print('Downloading package {pkg} from {url}...'.format(
pkg=self.real_name, url=download_url['url']))
download = urllib2.urlopen(download_url['url'])
except urllib2.HTTPError as http_error:
download = http_error
else:
self.used_url = download_url
self.as_string = download.read()
if not download_url['md5_digest']:
break
self.md5_sum = hashlib.md5(self.as_string).hexdigest()
if self.md5_sum == download_url['md5_digest']:
break
else:
if download.__class__ == urllib2.HTTPError:
raise download
raise DownloadFailed('Failed to downloas package {pkg}'
.format(pkg=self.real_name))
self.filename = self.used_url['filename']
self.url = self.used_url['url']
def extract_package(self, tmp_path):
"""
Extract the package contents into a directrory
Keyword arguments:
tmp_path -- directory where you want the package to be extracted
"""
as_file = StringIO.StringIO(self.as_string)
if self.filename[-3:] == 'zip':
with zipfile.ZipFile(as_file) as as_zipfile:
tmp_pkg = os.path.join(tmp_path, self.buildroot_name)
try:
os.makedirs(tmp_pkg)
except OSError as exception:
if exception.errno != errno.EEXIST:
print("ERROR: ", exception.message, file=sys.stderr)
return None, None
print('WARNING:', exception.message, file=sys.stderr)
print('Removing {pkg}...'.format(pkg=tmp_pkg))
shutil.rmtree(tmp_pkg)
os.makedirs(tmp_pkg)
as_zipfile.extractall(tmp_pkg)
else:
with tarfile.open(fileobj=as_file) as as_tarfile:
tmp_pkg = os.path.join(tmp_path, self.buildroot_name)
try:
os.makedirs(tmp_pkg)
except OSError as exception:
if exception.errno != errno.EEXIST:
print("ERROR: ", exception.message, file=sys.stderr)
return None, None
print('WARNING:', exception.message, file=sys.stderr)
print('Removing {pkg}...'.format(pkg=tmp_pkg))
shutil.rmtree(tmp_pkg)
os.makedirs(tmp_pkg)
as_tarfile.extractall(tmp_pkg)
tmp_extract = '{folder}/{name}-{version}'
self.tmp_extract = tmp_extract.format(
folder=tmp_pkg,
name=self.metadata_name,
version=self.version)
def load_setup(self):
"""
Loads the corresponding setup and store its metadata
"""
current_dir = os.getcwd()
os.chdir(self.tmp_extract)
sys.path.append(self.tmp_extract)
s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
setup = imp.load_module('setup', s_file, s_path, s_desc)
try:
self.setup_metadata = self.setup_args[self.metadata_name]
except KeyError:
# This means setup was not called which most likely mean that it is
# called through the if __name__ == '__main__' directive.
# In this case, we can only pray that it is called through a
# function called main() in setup.py.
setup.main([]) # Will raise AttributeError if not found
self.setup_metadata = self.setup_args[self.metadata_name]
# Here we must remove the module the hard way.
# We must do this because of a very sepcific case: if a package calls
# setup from the __main__ but does not come with a 'main()' function,
# for some reason setup.main([]) will successfully call the main
# function of a previous package...
sys.modules.pop('setup',None)
del setup
os.chdir(current_dir)
sys.path.remove(self.tmp_extract)
def get_requirements(self, pkg_folder):
"""
Retrieve dependencies from the metadata found in the setup.py script of
a pypi package.
Keyword Arguments:
pkg_folder -- location of the already created packages
"""
if 'install_requires' not in self.setup_metadata:
self.pkg_req = None
return set()
self.pkg_req = self.setup_metadata['install_requires']
self.pkg_req = [re.sub('([-.\w]+).*', r'\1', req)
for req in self.pkg_req]
req_not_found = self.pkg_req
self.pkg_req = map(pkg_buildroot_name, self.pkg_req)
pkg_tuples = zip(req_not_found, self.pkg_req)
# pkg_tuples is a list of tuples that looks like
# ('werkzeug','python-werkzeug') because I need both when checking if
# dependencies already exist or are already in the download list
req_not_found = set(
pkg[0] for pkg in pkg_tuples
if not os.path.isdir(pkg[1])
)
return req_not_found
def __create_mk_header(self):
"""
Create the header of the <package_name>.mk file
"""
header = ['#' * 80 + '\n']
header.append('#\n')
header.append('# {name}\n'.format(name=self.buildroot_name))
header.append('#\n')
header.append('#' * 80 + '\n')
header.append('\n')
return header
def __create_mk_download_info(self):
"""
Create the lines refering to the download information of the
<package_name>.mk file
"""
lines = []
version_line = '{name}_VERSION = {version}\n'.format(
name=self.mk_name,
version=self.version)
lines.append(version_line)
targz = self.filename.replace(
self.version,
'$({name}_VERSION)'.format(name=self.mk_name))
targz_line = '{name}_SOURCE = {filename}\n'.format(
name=self.mk_name,
filename=targz)
lines.append(targz_line)
if self.filename not in self.url:
# Sometimes the filename is in the url, sometimes it's not
site_url = self.url
else:
site_url = self.url[:self.url.find(self.filename)]
site_line = '{name}_SITE = {url}'.format(name=self.mk_name,
url=site_url)
site_line = site_line.rstrip('/') + '\n'
lines.append(site_line)
return lines
def __create_mk_setup(self):
"""
Create the line refering to the setup method of the package of the
<package_name>.mk file
There are two things you can use to make an installer
for a python package: distutils or setuptools
distutils comes with python but does not support dependencies.
distutils is mostly still there for backward support.
setuptools is what smart people use,
but it is not shipped with python :(
"""
lines = []
setup_type_line = '{name}_SETUP_TYPE = {method}\n'.format(
name=self.mk_name,
method=self.setup_metadata['method'])
lines.append(setup_type_line)
return lines
def __create_mk_license(self):
"""
Create the lines referring to the package's license informations of the
<package_name>.mk file
The license is found using the metadata from pypi.
In the metadata, the license can be found either with standard names in
the classifiers part or with naming from the packager in the "License"
part.
From the classifiers, the license is "translated" according to
buildroot standards if need be (i.e. from Apache Software License to
Apache-2.0).
From the License part, we cannot guess what formatting the packager
used. Hence, it is likely to be incorrect. (i.e. Apache License 2.0
instead of Apache-2.0).
The license's files are found by searching the package for files named
license or license.txt (case insensitive).
If more than one license file is found, the user is asked to select
which ones he wants to use.
"""
license_dict = {
'Apache Software License': 'Apache-2.0',
'BSD License': 'BSD',
'European Union Public Licence 1.0': 'EUPLv1.0',
'European Union Public Licence 1.1': 'EUPLv1.1',
"GNU General Public License": "GPL",
"GNU General Public License v2": "GPLv2",
"GNU General Public License v2 or later": "GPLv2+",
"GNU General Public License v3": "GPLv3",
"GNU General Public License v3 or later": "GPLv3+",
"GNU Lesser General Public License v2": "LGPLv2.1",
"GNU Lesser General Public License v2 or later": "LGPLv2.1+",
"GNU Lesser General Public License v3": "LGPLv3",
"GNU Lesser General Public License v3 or later": "LGPLv3+",
"GNU Library or Lesser General Public License": "LGPLv2",
"ISC License": "ISC",
"MIT License": "MIT",
"Mozilla Public License 1.0": "MPL-1.0",
"Mozilla Public License 1.1": "MPL-1.1",
"Mozilla Public License 2.0": "MPL-2.0",
"Zope Public License": "ZPL"
}
regexp = re.compile('^License :* *.* *:+ (.*)( \(.*\))?$')
classifiers_licenses = [regexp.sub(r"\1", lic)
for lic in self.metadata['info']['classifiers']
if regexp.match(lic)]
licenses = map(lambda x: license_dict[x] if x in license_dict else x,
classifiers_licenses)
lines = []
if not len(licenses):
print('WARNING: License has been set to "{license}". It is most'
' likely wrong, please change it if need be'.format(
license=', '.join(licenses)))
licenses = [self.metadata['info']['license']]
license_line = '{name}_LICENSE = {license}\n'.format(
name=self.mk_name,
license=', '.join(licenses))
lines.append(license_line)
filenames = ['LICENSE', 'LICENSE.TXT', 'COPYING', 'COPYING.TXT']
license_files = list(find_file_upper_case(filenames, self.tmp_extract))
license_files = [license.replace(self.tmp_extract, '')[1:]
for license in license_files]
if len(license_files) > 0:
if len(license_files) > 1:
print('More than one file found for license:',
', '.join(license_files))
license_files = [filename
for index, filename in enumerate(license_files)]
license_file_line = ('{name}_LICENSE_FILES ='
' {files}\n'.format(
name=self.mk_name,
files=' '.join(license_files)))
lines.append(license_file_line)
else:
print('WARNING: No license file found,'
' please specify it manually afterwards')
license_file_line = '# No license file found\n'
return lines
def __create_mk_requirements(self):
"""
Create the lines referring to the dependencies of the of the
<package_name>.mk file
Keyword Arguments:
pkg_name -- name of the package
pkg_req -- dependencies of the package
"""
lines = []
dependencies_line = ('{name}_DEPENDENCIES ='
' {reqs}\n'.format(
name=self.mk_name,
reqs=' '.join(self.pkg_req)))
lines.append(dependencies_line)
return lines
def create_package_mk(self):
"""
Create the lines corresponding to the <package_name>.mk file
"""
pkg_mk = '{name}.mk'.format(name=self.buildroot_name)
path_to_mk = os.path.join(self.pkg_dir, pkg_mk)
print('Creating {file}...'.format(file=path_to_mk))
lines = self.__create_mk_header()
lines += self.__create_mk_download_info()
lines += self.__create_mk_setup()
lines += self.__create_mk_license()
lines.append('\n')
lines.append('$(eval $(python-package))')
lines.append('\n')
with open(path_to_mk, 'w') as mk_file:
mk_file.writelines(lines)
def create_hash_file(self):
"""
Create the lines corresponding to the <package_name>.hash files
"""
pkg_hash = '{name}.hash'.format(name=self.buildroot_name)
path_to_hash = os.path.join(self.pkg_dir, pkg_hash)
print('Creating {filename}...'.format(filename=path_to_hash))
lines = []
if self.used_url['md5_digest']:
md5_comment = '# md5 from {url}\n'.format(url=self.metadata_url)
lines.append(md5_comment)
hash_line = '{method}\t{digest} {filename}\n'.format(
method='md5',
digest=self.used_url['md5_digest'],
filename=self.filename)
lines.append(hash_line)
sha256_comment = '# sha256 calculated by scanpypi\n'
lines.append(sha256_comment)
digest = hashlib.sha256(self.as_string).hexdigest()
hash_line = '{method}\t{digest} {filename}\n'.format(
method='sha256',
digest=digest,
filename=self.filename)
lines.append(hash_line)
with open(path_to_hash, 'w') as hash_file:
hash_file.writelines(lines)
def create_config_in(self):
"""
Creates the Config.in file of a package
"""
path_to_config = os.path.join(self.pkg_dir, 'Config.in')
print('Creating {file}...'.format(file=path_to_config))
lines = []
config_line = 'config BR2_PACKAGE_{name}\n'.format(
name=self.mk_name)
lines.append(config_line)
bool_line = '\tbool "{name}"\n'.format(name=self.buildroot_name)
lines.append(bool_line)
if self.pkg_req:
for dep in self.pkg_req:
dep_line = '\tselect BR2_PACKAGE_{req} # runtime\n'.format(
req=dep.upper().replace('-', '_'))
lines.append(dep_line)
lines.append('\thelp\n')
help_lines = textwrap.wrap(self.metadata['info']['summary'],
initial_indent='\t ',
subsequent_indent='\t ')
# \t + two spaces is 3 char long
help_lines.append('')
help_lines.append('\t ' + self.metadata['info']['home_page'])
help_lines = map(lambda x: x + '\n', help_lines)
lines += help_lines
with open(path_to_config, 'w') as config_file:
config_file.writelines(lines)
def main():
# Building the parser
parser = argparse.ArgumentParser(
description="Creates buildroot packages from the metadata of "
"an existing PyPI packages and include it "
"in menuconfig")
parser.add_argument("packages",
help="list of packages to be created",
nargs='+')
parser.add_argument("-o", "--output",
help="""
Output directory for packages.
Default is ./package
""",
default='./package')
args = parser.parse_args()
packages = list(set(args.packages))
# tmp_path is where we'll extract the files later
tmp_prefix = 'scanpypi-'
pkg_folder = args.output
tmp_path = tempfile.mkdtemp(prefix=tmp_prefix)
try:
for real_pkg_name in packages:
package = BuildrootPackage(real_pkg_name, pkg_folder)
print('buildroot package name for {}:'.format(package.real_name),
package.buildroot_name)
# First we download the package
# Most of the info we need can only be found inside the package
print('Package:', package.buildroot_name)
print('Fetching package', package.real_name)
try:
package.fetch_package_info()
except (urllib2.URLError, urllib2.HTTPError):
continue
if package.metadata_name.lower() == 'setuptools':
# setuptools imports itself, that does not work very well
# with the monkey path at the begining
print('Error: setuptools cannot be built using scanPyPI')
continue
try:
package.download_package()
except urllib2.HTTPError as error:
print('Error: {code} {reason}'.format(code=error.code,
reason=error.reason))
print('Error downloading package :', package.buildroot_name)
print()
continue
# extract the tarball
try:
package.extract_package(tmp_path)
except (tarfile.ReadError, zipfile.BadZipfile):
print('Error extracting package {}'.format(package.real_name))
print()
continue
# Loading the package install info from the package
try:
package.load_setup()
except ImportError as err:
if 'buildutils' in err.message:
print('This package needs buildutils')
else:
raise
continue
except AttributeError:
print('Error: Could not install package {pkg}'.format(
pkg=package.real_name))
continue
# Package requirement are an argument of the setup function
req_not_found = package.get_requirements(pkg_folder)
req_not_found = req_not_found.difference(packages)
packages += req_not_found
if req_not_found:
print('Added packages \'{pkgs}\' as dependencies of {pkg}'
.format(pkgs=", ".join(req_not_found),
pkg=package.buildroot_name))
print('Checking if package {name} already exists...'.format(
name=package.pkg_dir))
try:
os.makedirs(package.pkg_dir)
except OSError as exception:
if exception.errno != errno.EEXIST:
print("ERROR: ", exception.message, file=sys.stderr)
continue
print('Error: Package {name} already exists'
.format(name=package.pkg_dir))
del_pkg = raw_input(
'Do you want to delete existing package ? [y/N]')
if del_pkg.lower() == 'y':
shutil.rmtree(package.pkg_dir)
os.makedirs(package.pkg_dir)
else:
continue
package.create_package_mk()
package.create_hash_file()
package.create_config_in()
print()
# printing an empty line for visual confort
finally:
shutil.rmtree(tmp_path)
if __name__ == "__main__":
main()