From c6b08b1af2f9a68b62d7867f0a62c684303cce01 Mon Sep 17 00:00:00 2001 From: Andre Anjos <andre.dos.anjos@gmail.com> Date: Mon, 18 Nov 2013 22:05:32 +0100 Subject: [PATCH] Merge pypkg here --- .gitignore | 13 +- README.rst | 39 ++++ bootstrap.py | 184 +++++++++++++++++ buildout.cfg | 12 ++ setup.py | 1 - xbob/extension/__init__.py | 29 +-- xbob/extension/pkgconfig.py | 325 +++++++++++++++++++++++++++++++ xbob/extension/test_pkgconfig.py | 91 +++++++++ 8 files changed, 679 insertions(+), 15 deletions(-) create mode 100644 bootstrap.py create mode 100644 buildout.cfg create mode 100644 xbob/extension/pkgconfig.py create mode 100644 xbob/extension/test_pkgconfig.py diff --git a/.gitignore b/.gitignore index 7f59da6..a604073 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,16 @@ *~ *.swp *.pyc +bin +eggs +parts +.installed.cfg +.mr.developer.cfg *.egg-info -dist/ +src +develop-eggs +sphinx +dist +.nfs* +.gdb_history +build diff --git a/README.rst b/README.rst index d054bdc..3064da7 100644 --- a/README.rst +++ b/README.rst @@ -71,3 +71,42 @@ your ``buildout.cfg``. This includes, possibly, dependent projects. Currently, ``zc.buildout`` ignores the ``setup_requires`` entry on your ``setup.py`` file. The recipe above creates a new interpreter that hooks that package in and builds the project considering variables like ``prefixes`` into consideration. + +Python API to pkg-config +------------------------ + +This package alson contains a set of Pythonic bindings to the popular +pkg-config configuration utility. It allows distutils-based setup files to +query for libraries installed on the current system through that command line +utility. library. + +Using at your ``setup.py`` +========================== + +To use this package at your ``setup.py`` file, you will need to let distutils +know it needs it before importing it. You can achieve this with the following +trick:: + + from setuptools import dist + dist.Distribution(dict(setup_requires='xbob.extension')) + from xbob.extension.pkgconfig import pkgconfig + +.. note:: + + In this case, distutils should automatically download and install this + package on the environment it is required to setup other package. + +After inclusion, you can just instantiate an object of type ``pkgconfig``:: + + >>> zlib = pkgconfig('zlib') + >>> zlib.version # doctest: SKIP + 1.2.8 + >>> zlib.include_directories() # doctest: SKIP + ['/usr/include'] + >>> zlib.library_dirs # doctest: SKIP + ['/usr/lib'] + >>> zlib > '1.2.6' + True + >>> zlib > '1.2.10' + False + diff --git a/bootstrap.py b/bootstrap.py new file mode 100644 index 0000000..3995f07 --- /dev/null +++ b/bootstrap.py @@ -0,0 +1,184 @@ +############################################################################## +# +# Copyright (c) 2006 Zope Foundation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# +############################################################################## +"""Bootstrap a buildout-based project + +Simply run this script in a directory containing a buildout.cfg. +The script accepts buildout command-line options, so you can +use the -c option to specify an alternate configuration file. +""" + +import os +import shutil +import sys +import tempfile + +from optparse import OptionParser + +tmpeggs = tempfile.mkdtemp() + +usage = '''\ +[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options] + +Bootstraps a buildout-based project. + +Simply run this script in a directory containing a buildout.cfg, using the +Python that you want bin/buildout to use. + +Note that by using --find-links to point to local resources, you can keep +this script from going over the network. +''' + +parser = OptionParser(usage=usage) +parser.add_option("-v", "--version", help="use a specific zc.buildout version") + +parser.add_option("-t", "--accept-buildout-test-releases", + dest='accept_buildout_test_releases', + action="store_true", default=False, + help=("Normally, if you do not specify a --version, the " + "bootstrap script and buildout gets the newest " + "*final* versions of zc.buildout and its recipes and " + "extensions for you. If you use this flag, " + "bootstrap and buildout will get the newest releases " + "even if they are alphas or betas.")) +parser.add_option("-c", "--config-file", + help=("Specify the path to the buildout configuration " + "file to be used.")) +parser.add_option("-f", "--find-links", + help=("Specify a URL to search for buildout releases")) + + +options, args = parser.parse_args() + +###################################################################### +# load/install setuptools + +to_reload = False +try: + import pkg_resources + import setuptools +except ImportError: + ez = {} + + try: + from urllib.request import urlopen + except ImportError: + from urllib2 import urlopen + + # XXX use a more permanent ez_setup.py URL when available. + exec(urlopen('https://bitbucket.org/pypa/setuptools/raw/0.7.2/ez_setup.py' + ).read(), ez) + setup_args = dict(to_dir=tmpeggs, download_delay=0) + ez['use_setuptools'](**setup_args) + + if to_reload: + reload(pkg_resources) + import pkg_resources + # This does not (always?) update the default working set. We will + # do it. + for path in sys.path: + if path not in pkg_resources.working_set.entries: + pkg_resources.working_set.add_entry(path) + +###################################################################### +# Try to best guess the version of buildout given setuptools +if options.version is None: + + try: + from distutils.version import LooseVersion + package = pkg_resources.require('setuptools')[0] + v = LooseVersion(package.version) + if v < LooseVersion('0.7'): + options.version = '2.1.1' + except: + pass + +###################################################################### +# Install buildout + +ws = pkg_resources.working_set + +cmd = [sys.executable, '-c', + 'from setuptools.command.easy_install import main; main()', + '-mZqNxd', tmpeggs] + +find_links = os.environ.get( + 'bootstrap-testing-find-links', + options.find_links or + ('http://downloads.buildout.org/' + if options.accept_buildout_test_releases else None) + ) +if find_links: + cmd.extend(['-f', find_links]) + +setuptools_path = ws.find( + pkg_resources.Requirement.parse('setuptools')).location + +requirement = 'zc.buildout' +version = options.version +if version is None and not options.accept_buildout_test_releases: + # Figure out the most recent final version of zc.buildout. + import setuptools.package_index + _final_parts = '*final-', '*final' + + def _final_version(parsed_version): + for part in parsed_version: + if (part[:1] == '*') and (part not in _final_parts): + return False + return True + index = setuptools.package_index.PackageIndex( + search_path=[setuptools_path]) + if find_links: + index.add_find_links((find_links,)) + req = pkg_resources.Requirement.parse(requirement) + if index.obtain(req) is not None: + best = [] + bestv = None + for dist in index[req.project_name]: + distv = dist.parsed_version + if _final_version(distv): + if bestv is None or distv > bestv: + best = [dist] + bestv = distv + elif distv == bestv: + best.append(dist) + if best: + best.sort() + version = best[-1].version +if version: + requirement = '=='.join((requirement, version)) +cmd.append(requirement) + +import subprocess +if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0: + raise Exception( + "Failed to execute command:\n%s", + repr(cmd)[1:-1]) + +###################################################################### +# Import and run buildout + +ws.add_entry(tmpeggs) +ws.require(requirement) +import zc.buildout.buildout + +if not [a for a in args if '=' not in a]: + args.append('bootstrap') + +# if -c was provided, we push it back into args for buildout' main function +if options.config_file is not None: + args[0:0] = ['-c', options.config_file] + +zc.buildout.buildout.main(args) +shutil.rmtree(tmpeggs) + diff --git a/buildout.cfg b/buildout.cfg new file mode 100644 index 0000000..e929e88 --- /dev/null +++ b/buildout.cfg @@ -0,0 +1,12 @@ +; vim: set fileencoding=utf-8 : +; Andre Anjos <andre.anjos@idiap.ch> +; Mon 16 Apr 08:29:18 2012 CEST + +[buildout] +parts = scripts +develop = . +eggs = xbob.extension + ipdb + +[scripts] +recipe = xbob.buildout:scripts diff --git a/setup.py b/setup.py index 4c99cce..fa95e83 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ setup( install_requires=[ 'setuptools', - 'pypkg', ], classifiers = [ diff --git a/xbob/extension/__init__.py b/xbob/extension/__init__.py index a5e343d..7c8ebf1 100644 --- a/xbob/extension/__init__.py +++ b/xbob/extension/__init__.py @@ -3,13 +3,15 @@ # Andre Anjos <andre.anjos@idiap.ch> # Mon 28 Jan 2013 16:40:27 CET -"""A custom build class for Bob/Python extensions +"""A custom build class for Pkg-config based extensions """ import platform -from pypkg import pkgconfig +from .pkgconfig import pkgconfig from distutils.extension import Extension as DistutilsExtension +__version__ = __import__('pkg_resources').require('xbob.extension')[0].version + def uniq(seq): """Uniqu-fy preserving order""" @@ -37,19 +39,20 @@ def check_packages(packages): from re import split + used = set() retval = [] for requirement in uniq(packages): splitreq = split(r'\s*(?P<cmp>[<>=]+)\s*', requirement) - if len(splitreq) == 1: # just package name + if len(splitreq) not in (1, 3): - p = pkgconfig(splitreq[0]) + raise RuntimeError("cannot parse requirement `%s'", requirement) - elif len(splitreq) == 3: # package + version number + p = pkgconfig(splitreq[0]) - p = pkgconfig(splitreq[0]) + if len(splitreq) == 3: # package + version number if splitreq[1] == '>': assert p > splitreq[2], "%s version is not > `%s'" % (p, splitreq[2]) @@ -64,17 +67,17 @@ def check_packages(packages): else: raise RuntimeError("cannot parse requirement `%s'", requirement) - else: - - raise RuntimeError("cannot parse requirement `%s'", requirement) - retval.append(p) + if p.name in used: + raise RuntimeError("package `%s' had already been requested - cannot (currently) handle recurring requirements") + used.add(p.name) + return retval class Extension(DistutilsExtension): - """Extension building with Bob/Python bindings. + """Extension building with pkg-config packages. See the documentation for :py:class:`distutils.extension.Extension` for more details on input parameters. @@ -83,8 +86,8 @@ class Extension(DistutilsExtension): def __init__(self, *args, **kwargs): """Initialize the extension with parameters. - Bob/Python adds a single parameter to the standard arguments of the - constructor: + Pkg-config extensions adds a single parameter to the standard arguments of + the constructor: pkgconfig : [list] diff --git a/xbob/extension/pkgconfig.py b/xbob/extension/pkgconfig.py new file mode 100644 index 0000000..92a15bb --- /dev/null +++ b/xbob/extension/pkgconfig.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python +# vim: set fileencoding=utf-8 : +# Andre Anjos <andre.anjos@idiap.ch> +# Wed 16 Oct 10:08:42 2013 CEST + +import os +import subprocess +import logging + +def uniq(seq, idfun=None): + """Very fast, order preserving uniq function""" + + # order preserving + if idfun is None: + def idfun(x): return x + seen = {} + result = [] + for item in seq: + marker = idfun(item) + # in old Python versions: + # if seen.has_key(marker) + # but in new ones: + if marker in seen: continue + seen[marker] = 1 + result.append(item) + return result + +def call_pkgconfig(cmd, paths=None): + """Runs a command as a subprocess and raises if that does not work + + Returns the exit status, stdout and stderr. + """ + + # if the user has passed their own paths, add it to the environment + env = os.environ + if paths is not None: + env = os.environ.copy() + var = os.pathsep.join(paths) + old = env.get('PKG_CONFIG_PATH', False) + env['PKG_CONFIG_PATH'] = os.pathsep.join([var, old]) if old else var + + # calls the lua creation script using the parameters + cmd = ['pkg-config'] + [str(k) for k in cmd] + subproc = subprocess.Popen( + cmd, + env=env, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + + logging.debug("Running `%s'" % (" ".join(cmd),)) + stdout, stderr = subproc.communicate() + + # always print the stdout + logger = logging.getLogger('pkgconfig') + for k in stdout.split('\n'): + if k: logger.debug(k) + + # handles py3k string conversion, if necessary + if isinstance(stdout, bytes) and not isinstance(stdout, str): + stdout = stdout.decode('utf8') + + if isinstance(stderr, bytes) and not isinstance(stderr, str): + stderr = stderr.decode('utf8') + + return subproc.returncode, stdout, stderr + +class pkgconfig: + """A class for capturing configuration information from pkg-config + + Example usage: + + .. doctest:: + :options: +NORMALIZE_WHITESPACE +ELLIPSIS + + >>> glibc = pkgconfig('glibc') + >>> glibc.include_directories() # doctest: SKIP + ['/usr/include'] + >>> glibc.library_directories() # doctest: SKIP + ['/usr/lib'] + + If the package does not exist, a RuntimeError is raised. All calls to any + methods of a ``pkgconfig`` object are translated into a subprocess call that + queries for that specific information. If ``pkg-config`` fails, a + RuntimeError is raised. + """ + + def __init__(self, name, paths=None): + """Constructor + + Parameters: + + name + The name of the package of interest, as you would pass on the command + line + + extra_paths + Search paths to be added to the environment's PKG_CONFIG_PATH to search + for packages. + + Equivalent command line version: + + .. code-block:: sh + + $ PKG_CONFIG_PATH=<paths> pkg-config <name> + + """ + + status, stdout, stderr = call_pkgconfig(['--modversion', name], paths) + + if status != 0: + raise RuntimeError("pkg-config package `%s' was not found" % name) + + self.name = name + self.version = stdout.strip() + self.paths = paths + + def __xcall__(self, cmd): + """Calls call_pkgconfig() with self.package and self.paths""" + + return call_pkgconfig(cmd + [self.name], self.paths) + + def __cmp__(self, other): + """Compares this package with a version number + + We create a new ``distutils.version.LooseVersion`` object out of your input + argument and then, compare it to our own version, returning the result. + + Returns an integer smaller than zero if this package's version number is + smaller than the provided value. Returns zero in case of a match and + greater than zero in the other case. + """ + + from distutils.version import LooseVersion + return cmp(self.version, LooseVersion(other)) + + def include_directories(self): + """Returns a pre-processed list containing include directories. + + Equivalent command line version: + + .. code-block:: sh + + $ PKG_CONFIG_PATH=<paths> pkg-config --cflags-only-I <name> + + """ + + status, stdout, stderr = self.__xcall__(['--cflags-only-I']) + + if status != 0: + raise RuntimeError("error querying --cflags-only-I for package `%s': %s" % (self.package, stderr)) + + retval = [] + for token in stdout.split(): + retval.append(token[2:]) + + return uniq(retval) + + def cflags_other(self): + """Returns a pre-processed dictionary containing compilation options. + + Equivalent command line version: + + .. code-block:: sh + + $ PKG_CONFIG_PATH=<paths> pkg-config --cflags-only-other <name> + + The returned dictionary contains two entries ``extra_compile_args`` and + ``define_macros``. The ``define_macros`` entries are ready for deployment + in the ``setup()`` function of your package. + """ + + status, stdout, stderr = self.__xcall__(['--cflags-only-other']) + + if status != 0: + raise RuntimeError("error querying --cflags-only-other for package `%s': %s" % (self.package, stderr)) + + flag_map = { + '-D': 'define_macros', + } + + kw = {} + + for token in stdout.split(): + if token[:2] in flag_map: + kw.setdefault(flag_map.get(token[:2]), []).append(token[2:]) + + else: # throw others to extra_link_args + kw.setdefault('extra_compile_args', []).append(token) + + # make it uniq + for k, v in kw.items(): kw[k] = uniq(v) + + # for macros, separate them so they can be plugged on C/C++ extensions + if 'define_macros' in kw: + for k, string in enumerate(kw['define_macros']): + if string.find('=') != -1: + kw['define_macros'][k] = string.split('=', 2) + else: + kw['define_macros'][k] = (string, None) + + return kw + + def libraries(self): + """Returns a pre-processed list containing libraries to link against + + Equivalent command line version: + + .. code-block:: sh + + $ PKG_CONFIG_PATH=<paths> pkg-config --libs-only-l <name> + + """ + + status, stdout, stderr = self.__xcall__(['--libs-only-l']) + + if status != 0: + raise RuntimeError("error querying --libs-only-l for package `%s': %s" % (self.package, stderr)) + + retval = [] + for token in stdout.split(): + retval.append(token[2:]) + + return uniq(retval) + + def library_directories(self): + """Returns a pre-processed list containing library directories. + + Equivalent command line version: + + .. code-block:: sh + + $ PKG_CONFIG_PATH=<paths> pkg-config --libs-only-L <name> + + """ + + status, stdout, stderr = self.__xcall__(['--libs-only-L']) + + if status != 0: + raise RuntimeError("error querying --libs-only-L for package `%s': %s" % (self.package, stderr)) + + retval = [] + for token in stdout.split(): + retval.append(token[2:]) + + return uniq(retval) + + def extra_link_args(self): + """Returns a pre-processed list containing extra link arguments. + + Equivalent command line version: + + .. code-block:: sh + + $ PKG_CONFIG_PATH=<paths> pkg-config --libs-only-other <name> + + """ + + status, stdout, stderr = self.__xcall__(['--libs-only-other']) + + if status != 0: + raise RuntimeError("error querying --libs-only-other for package `%s': %s" % (self.package, stderr)) + + return stdout.strip().split() + + def variable_names(self): + """Returns a list with all variable names know to this package + + Equivalent command line version: + + .. code-block:: sh + + $ PKG_CONFIG_PATH=<paths> pkg-config --print-variables <name> + + """ + + status, stdout, stderr = self.__xcall__(['--print-variables']) + + if status != 0: + raise RuntimeError("error querying --print-variables for package `%s': %s" % (self.package, stderr)) + + return stdout.strip().split() + + def variable(self, name): + """Returns a variable with a specific name (if it exists) + + Equivalent command line version: + + .. code-block:: sh + + $ PKG_CONFIG_PATH=<paths> pkg-config --variable=<variable-name> <name> + + .. warning:: + + If a variable does not exist in a package, pkg-config does not signal an + error. Instead, it returns an empty string. So, do we. + """ + + status, stdout, stderr = self.__xcall__(['--variable=%s' % name]) + + if status != 0: + raise RuntimeError("error querying --variable=%s for package `%s': %s" % (name, self.package, stderr)) + + return stdout.strip() + + def package_macros(self): + """Returns package availability and version number macros + + This method returns a python list with 2 macros indicating package + availability and a version number, using standard GNU compatible names. For + example, if the package is named ``foo`` and its version is ``1.4``, this + command would return: + + .. code-block:: sh + + >>> foo = pkgconfig('foo') + >>> foo.package_macros() + [('HAVE_FOO', '1'), ('FOO_VERSION', '"1.4"')] + + """ + from re import sub + NAME = sub(r'[\.\-\s]', '_', self.name.upper()) + return [('HAVE_' + NAME, '1'), (NAME + '_VERSION', '"%s"' % self.version)] + +__all__ = ['pkgconfig'] diff --git a/xbob/extension/test_pkgconfig.py b/xbob/extension/test_pkgconfig.py new file mode 100644 index 0000000..d65133f --- /dev/null +++ b/xbob/extension/test_pkgconfig.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# vim: set fileencoding=utf-8 : +# Andre Anjos <andre.anjos@idiap.ch> +# Wed 16 Oct 12:16:49 2013 + +"""Tests for pkgconfig +""" + +import nose +from .pkgconfig import pkgconfig + +test_package = 'zlib' + +def test_detect_ok(): + pkg = pkgconfig(test_package) + nose.tools.eq_(pkg.name, test_package) + assert pkg.version + #print pkg.name, pkg.version + +@nose.tools.raises(RuntimeError) +def test_detect_not_ok(): + pkg = pkgconfig('foobarfoo') + +def test_include_directories(): + pkg = pkgconfig(test_package) + obj = pkg.include_directories() + assert isinstance(obj, list) + assert obj + for k in obj: + assert k.find('-I') != 0 + #print obj + +def test_cflags_other(): + pkg = pkgconfig('QtCore') + obj = pkg.cflags_other() + assert obj['define_macros'] + assert isinstance(obj['define_macros'], list) + assert isinstance(obj['define_macros'][0], tuple) + assert isinstance(obj, dict) + #print obj + +def test_libraries(): + pkg = pkgconfig(test_package) + obj = pkg.libraries() + assert isinstance(obj, list) + assert obj + for k in obj: + assert k.find('-l') != 0 + #print obj + +def test_library_directories(): + pkg = pkgconfig(test_package) + obj = pkg.library_directories() + assert isinstance(obj, list) + assert obj + for k in obj: + assert k.find('-L') != 0 + #print obj + +def test_extra_link_args(): + pkg = pkgconfig(test_package) + obj = pkg.extra_link_args() + assert isinstance(obj, list) + #print obj + +def test_variable_names(): + pkg = pkgconfig(test_package) + obj = pkg.variable_names() + assert isinstance(obj, list) + assert obj + #print obj + +def test_variable(): + pkg = pkgconfig(test_package) + names = pkg.variable_names() + assert isinstance(names, list) + assert names + v = pkg.variable(names[0]) + assert v + #print v + +def test_macros(): + pkg = pkgconfig(test_package) + macros = pkg.package_macros() + assert isinstance(macros, list) + assert macros + assert macros[0][0].find('HAVE_') == 0 + assert macros[0][1] == '1' + assert macros[1][0].find('_VERSION') > 0 + assert macros[1][1].find('"') == 0 + assert macros[1][1].rfind('"') == (len(macros[1][1]) - 1) -- GitLab