Merge pypkg here

.gitignore

 *~
 *.swp
 *.pyc
+bin
+eggs
+parts
+.installed.cfg
+.mr.developer.cfg
 *.egg-info
-dist/
+src
+develop-eggs
+sphinx
+dist
+.nfs*
+.gdb_history
+build

README.rst

@@ -71,3 +71,42 @@ your ``buildout.cfg``. This includes, possibly, dependent projects. Currently,
 ``zc.buildout`` ignores the ``setup_requires`` entry on your ``setup.py`` file.
 The recipe above creates a new interpreter that hooks that package in and
 builds the project considering variables like ``prefixes`` into consideration.
+
+Python API to pkg-config
+------------------------
+
+This package alson contains a set of Pythonic bindings to the popular
+pkg-config configuration utility. It allows distutils-based setup files to
+query for libraries installed on the current system through that command line
+utility.  library.
+
+Using at your ``setup.py``
+==========================
+
+To use this package at your ``setup.py`` file, you will need to let distutils
+know it needs it before importing it. You can achieve this with the following
+trick::
+
+  from setuptools import dist
+  dist.Distribution(dict(setup_requires='xbob.extension'))
+  from xbob.extension.pkgconfig import pkgconfig
+
+.. note::
+
+   In this case, distutils should automatically download and install this
+   package on the environment it is required to setup other package.
+
+After inclusion, you can just instantiate an object of type ``pkgconfig``::
+
+  >>> zlib = pkgconfig('zlib')
+  >>> zlib.version # doctest: SKIP
+  1.2.8
+  >>> zlib.include_directories() # doctest: SKIP
+  ['/usr/include']
+  >>> zlib.library_dirs # doctest: SKIP
+  ['/usr/lib']
+  >>> zlib > '1.2.6'
+  True
+  >>> zlib > '1.2.10'
+  False
+

bootstrap.py

+##############################################################################
+#
+# Copyright (c) 2006 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Bootstrap a buildout-based project
+
+Simply run this script in a directory containing a buildout.cfg.
+The script accepts buildout command-line options, so you can
+use the -c option to specify an alternate configuration file.
+"""
+
+import os
+import shutil
+import sys
+import tempfile
+
+from optparse import OptionParser
+
+tmpeggs = tempfile.mkdtemp()
+
+usage = '''\
+[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
+
+Bootstraps a buildout-based project.
+
+Simply run this script in a directory containing a buildout.cfg, using the
+Python that you want bin/buildout to use.
+
+Note that by using --find-links to point to local resources, you can keep
+this script from going over the network.
+'''
+
+parser = OptionParser(usage=usage)
+parser.add_option("-v", "--version", help="use a specific zc.buildout version")
+
+parser.add_option("-t", "--accept-buildout-test-releases",
+                  dest='accept_buildout_test_releases',
+                  action="store_true", default=False,
+                  help=("Normally, if you do not specify a --version, the "
+                        "bootstrap script and buildout gets the newest "
+                        "*final* versions of zc.buildout and its recipes and "
+                        "extensions for you.  If you use this flag, "
+                        "bootstrap and buildout will get the newest releases "
+                        "even if they are alphas or betas."))
+parser.add_option("-c", "--config-file",
+                  help=("Specify the path to the buildout configuration "
+                        "file to be used."))
+parser.add_option("-f", "--find-links",
+                  help=("Specify a URL to search for buildout releases"))
+
+
+options, args = parser.parse_args()
+
+######################################################################
+# load/install setuptools
+
+to_reload = False
+try:
+    import pkg_resources
+    import setuptools
+except ImportError:
+    ez = {}
+
+    try:
+        from urllib.request import urlopen
+    except ImportError:
+        from urllib2 import urlopen
+
+    # XXX use a more permanent ez_setup.py URL when available.
+    exec(urlopen('https://bitbucket.org/pypa/setuptools/raw/0.7.2/ez_setup.py'
+                ).read(), ez)
+    setup_args = dict(to_dir=tmpeggs, download_delay=0)
+    ez['use_setuptools'](**setup_args)
+
+    if to_reload:
+        reload(pkg_resources)
+    import pkg_resources
+    # This does not (always?) update the default working set.  We will
+    # do it.
+    for path in sys.path:
+        if path not in pkg_resources.working_set.entries:
+            pkg_resources.working_set.add_entry(path)
+
+######################################################################
+# Try to best guess the version of buildout given setuptools
+if options.version is None:
+
+  try:
+    from distutils.version import LooseVersion
+    package = pkg_resources.require('setuptools')[0]
+    v = LooseVersion(package.version)
+    if v < LooseVersion('0.7'):
+      options.version = '2.1.1'
+  except:
+    pass
+
+######################################################################
+# Install buildout
+
+ws = pkg_resources.working_set
+
+cmd = [sys.executable, '-c',
+       'from setuptools.command.easy_install import main; main()',
+       '-mZqNxd', tmpeggs]
+
+find_links = os.environ.get(
+    'bootstrap-testing-find-links',
+    options.find_links or
+    ('http://downloads.buildout.org/'
+     if options.accept_buildout_test_releases else None)
+    )
+if find_links:
+    cmd.extend(['-f', find_links])
+
+setuptools_path = ws.find(
+    pkg_resources.Requirement.parse('setuptools')).location
+
+requirement = 'zc.buildout'
+version = options.version
+if version is None and not options.accept_buildout_test_releases:
+    # Figure out the most recent final version of zc.buildout.
+    import setuptools.package_index
+    _final_parts = '*final-', '*final'
+
+    def _final_version(parsed_version):
+        for part in parsed_version:
+            if (part[:1] == '*') and (part not in _final_parts):
+                return False
+        return True
+    index = setuptools.package_index.PackageIndex(
+        search_path=[setuptools_path])
+    if find_links:
+        index.add_find_links((find_links,))
+    req = pkg_resources.Requirement.parse(requirement)
+    if index.obtain(req) is not None:
+        best = []
+        bestv = None
+        for dist in index[req.project_name]:
+            distv = dist.parsed_version
+            if _final_version(distv):
+                if bestv is None or distv > bestv:
+                    best = [dist]
+                    bestv = distv
+                elif distv == bestv:
+                    best.append(dist)
+        if best:
+            best.sort()
+            version = best[-1].version
+if version:
+    requirement = '=='.join((requirement, version))
+cmd.append(requirement)
+
+import subprocess
+if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0:
+    raise Exception(
+        "Failed to execute command:\n%s",
+        repr(cmd)[1:-1])
+
+######################################################################
+# Import and run buildout
+
+ws.add_entry(tmpeggs)
+ws.require(requirement)
+import zc.buildout.buildout
+
+if not [a for a in args if '=' not in a]:
+    args.append('bootstrap')
+
+# if -c was provided, we push it back into args for buildout' main function
+if options.config_file is not None:
+    args[0:0] = ['-c', options.config_file]
+
+zc.buildout.buildout.main(args)
+shutil.rmtree(tmpeggs)
+

buildout.cfg

+; vim: set fileencoding=utf-8 :
+; Andre Anjos <andre.anjos@idiap.ch>
+; Mon 16 Apr 08:29:18 2012 CEST
+
+[buildout]
+parts = scripts
+develop = .
+eggs = xbob.extension
+       ipdb
+
+[scripts]
+recipe = xbob.buildout:scripts

setup.py

@@ -28,7 +28,6 @@ setup(
 
     install_requires=[
       'setuptools',
-      'pypkg',
       ],
 
     classifiers = [

xbob/extension/__init__.py

@@ -3,13 +3,15 @@
 # Andre Anjos <andre.anjos@idiap.ch>
 # Mon 28 Jan 2013 16:40:27 CET
 
-"""A custom build class for Bob/Python extensions
+"""A custom build class for Pkg-config based extensions
 """
 
 import platform
-from pypkg import pkgconfig
+from .pkgconfig import pkgconfig
 from distutils.extension import Extension as DistutilsExtension
 
+__version__ = __import__('pkg_resources').require('xbob.extension')[0].version
+
 def uniq(seq):
   """Uniqu-fy preserving order"""
 
@@ -37,19 +39,20 @@ def check_packages(packages):
 
   from re import split
 
+  used = set()
   retval = []
 
   for requirement in uniq(packages):
 
     splitreq = split(r'\s*(?P<cmp>[<>=]+)\s*', requirement)
 
-    if len(splitreq) == 1: # just package name
+    if len(splitreq) not in (1, 3):
 
-      p = pkgconfig(splitreq[0])
+      raise RuntimeError("cannot parse requirement `%s'", requirement)
 
-    elif len(splitreq) == 3: # package + version number
+    p = pkgconfig(splitreq[0])
 
-      p = pkgconfig(splitreq[0]) 
+    if len(splitreq) == 3: # package + version number
 
       if splitreq[1] == '>': 
         assert p > splitreq[2], "%s version is not > `%s'" % (p, splitreq[2])
@@ -64,17 +67,17 @@ def check_packages(packages):
       else:
         raise RuntimeError("cannot parse requirement `%s'", requirement)
 
-    else:
-
-      raise RuntimeError("cannot parse requirement `%s'", requirement)
-
     retval.append(p)
 
+    if p.name in used:
+      raise RuntimeError("package `%s' had already been requested - cannot (currently) handle recurring requirements")
+    used.add(p.name)
+
   return retval
 
 
 class Extension(DistutilsExtension):
-  """Extension building with Bob/Python bindings.
+  """Extension building with pkg-config packages.
 
   See the documentation for :py:class:`distutils.extension.Extension` for more
   details on input parameters.
@@ -83,8 +86,8 @@ class Extension(DistutilsExtension):
   def __init__(self, *args, **kwargs):
     """Initialize the extension with parameters.
 
-    Bob/Python adds a single parameter to the standard arguments of the
-    constructor:
+    Pkg-config extensions adds a single parameter to the standard arguments of
+    the constructor:
 
     pkgconfig : [list]
 

xbob/extension/pkgconfig.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Andre Anjos <andre.anjos@idiap.ch>
+# Wed 16 Oct 10:08:42 2013 CEST
+
+import os
+import subprocess
+import logging
+
+def uniq(seq, idfun=None):
+  """Very fast, order preserving uniq function"""
+
+  # order preserving
+  if idfun is None:
+      def idfun(x): return x
+  seen = {}
+  result = []
+  for item in seq:
+      marker = idfun(item)
+      # in old Python versions:
+      # if seen.has_key(marker)
+      # but in new ones:
+      if marker in seen: continue
+      seen[marker] = 1
+      result.append(item)
+  return result
+
+def call_pkgconfig(cmd, paths=None):
+  """Runs a command as a subprocess and raises if that does not work
+  
+  Returns the exit status, stdout and stderr.
+  """
+
+  # if the user has passed their own paths, add it to the environment
+  env = os.environ
+  if paths is not None:
+    env = os.environ.copy()
+    var = os.pathsep.join(paths)
+    old = env.get('PKG_CONFIG_PATH', False)
+    env['PKG_CONFIG_PATH'] = os.pathsep.join([var, old]) if old else var
+
+  # calls the lua creation script using the parameters
+  cmd = ['pkg-config'] + [str(k) for k in cmd]
+  subproc = subprocess.Popen(
+      cmd,
+      env=env,
+      stderr=subprocess.PIPE,
+      stdout=subprocess.PIPE
+      )
+
+  logging.debug("Running `%s'" % (" ".join(cmd),))
+  stdout, stderr = subproc.communicate()
+
+  # always print the stdout
+  logger = logging.getLogger('pkgconfig')
+  for k in stdout.split('\n'):
+    if k: logger.debug(k)
+
+  # handles py3k string conversion, if necessary
+  if isinstance(stdout, bytes) and not isinstance(stdout, str):
+    stdout = stdout.decode('utf8')
+
+  if isinstance(stderr, bytes) and not isinstance(stderr, str):
+    stderr = stderr.decode('utf8')
+
+  return subproc.returncode, stdout, stderr
+
+class pkgconfig:
+  """A class for capturing configuration information from pkg-config
+
+  Example usage:
+
+    .. doctest::
+       :options: +NORMALIZE_WHITESPACE +ELLIPSIS
+
+       >>> glibc = pkgconfig('glibc')
+       >>> glibc.include_directories() # doctest: SKIP
+       ['/usr/include']
+       >>> glibc.library_directories() # doctest: SKIP
+       ['/usr/lib']
+
+  If the package does not exist, a RuntimeError is raised. All calls to any 
+  methods of a ``pkgconfig`` object are translated into a subprocess call that
+  queries for that specific information. If ``pkg-config`` fails, a
+  RuntimeError is raised.
+  """
+
+  def __init__(self, name, paths=None):
+    """Constructor
+
+    Parameters:
+
+    name
+      The name of the package of interest, as you would pass on the command
+      line
+
+    extra_paths
+      Search paths to be added to the environment's PKG_CONFIG_PATH to search
+      for packages.
+    
+    Equivalent command line version:
+
+    .. code-block:: sh
+
+       $ PKG_CONFIG_PATH=<paths> pkg-config <name>
+
+    """
+
+    status, stdout, stderr = call_pkgconfig(['--modversion', name], paths)
+
+    if status != 0:
+      raise RuntimeError("pkg-config package `%s' was not found" % name)
+
+    self.name = name
+    self.version = stdout.strip()
+    self.paths = paths
+
+  def __xcall__(self, cmd):
+    """Calls call_pkgconfig() with self.package and self.paths"""
+
+    return call_pkgconfig(cmd + [self.name], self.paths)
+
+  def __cmp__(self, other):
+    """Compares this package with a version number
+
+    We create a new ``distutils.version.LooseVersion`` object out of your input
+    argument and then, compare it to our own version, returning the result.
+
+    Returns an integer smaller than zero if this package's version number is
+    smaller than the provided value. Returns zero in case of a match and
+    greater than zero in the other case.
+    """
+
+    from distutils.version import LooseVersion
+    return cmp(self.version, LooseVersion(other))
+
+  def include_directories(self):
+    """Returns a pre-processed list containing include directories.
+    
+    Equivalent command line version:
+
+    .. code-block:: sh
+
+       $ PKG_CONFIG_PATH=<paths> pkg-config --cflags-only-I <name>
+
+    """
+
+    status, stdout, stderr = self.__xcall__(['--cflags-only-I'])
+
+    if status != 0: 
+      raise RuntimeError("error querying --cflags-only-I for package `%s': %s" % (self.package, stderr))
+
+    retval = []
+    for token in stdout.split():
+      retval.append(token[2:])
+
+    return uniq(retval)
+
+  def cflags_other(self):
+    """Returns a pre-processed dictionary containing compilation options.
+   
+    Equivalent command line version:
+
+    .. code-block:: sh
+
+       $ PKG_CONFIG_PATH=<paths> pkg-config --cflags-only-other <name>
+
+    The returned dictionary contains two entries ``extra_compile_args`` and
+    ``define_macros``. The ``define_macros`` entries are ready for deployment 
+    in the ``setup()`` function of your package.
+    """
+
+    status, stdout, stderr = self.__xcall__(['--cflags-only-other'])
+
+    if status != 0: 
+      raise RuntimeError("error querying --cflags-only-other for package `%s': %s" % (self.package, stderr))
+
+    flag_map = {
+        '-D': 'define_macros',
+        }
+
+    kw = {}
+
+    for token in stdout.split():
+      if token[:2] in flag_map:
+        kw.setdefault(flag_map.get(token[:2]), []).append(token[2:])
+
+      else: # throw others to extra_link_args
+        kw.setdefault('extra_compile_args', []).append(token)
+
+    # make it uniq
+    for k, v in kw.items(): kw[k] = uniq(v)
+
+    # for macros, separate them so they can be plugged on C/C++ extensions
+    if 'define_macros' in kw:
+      for k, string in enumerate(kw['define_macros']):
+        if string.find('=') != -1:
+          kw['define_macros'][k] = string.split('=', 2)
+        else:
+          kw['define_macros'][k] = (string, None)
+
+    return kw
+
+  def libraries(self):
+    """Returns a pre-processed list containing libraries to link against
+    
+    Equivalent command line version:
+
+    .. code-block:: sh
+
+       $ PKG_CONFIG_PATH=<paths> pkg-config --libs-only-l <name>
+
+    """
+
+    status, stdout, stderr = self.__xcall__(['--libs-only-l'])
+
+    if status != 0: 
+      raise RuntimeError("error querying --libs-only-l for package `%s': %s" % (self.package, stderr))
+
+    retval = []
+    for token in stdout.split():
+      retval.append(token[2:])
+
+    return uniq(retval)
+
+  def library_directories(self):
+    """Returns a pre-processed list containing library directories.
+    
+    Equivalent command line version:
+
+    .. code-block:: sh
+
+       $ PKG_CONFIG_PATH=<paths> pkg-config --libs-only-L <name>
+
+    """
+
+    status, stdout, stderr = self.__xcall__(['--libs-only-L'])
+
+    if status != 0: 
+      raise RuntimeError("error querying --libs-only-L for package `%s': %s" % (self.package, stderr))
+
+    retval = []
+    for token in stdout.split():
+      retval.append(token[2:])
+
+    return uniq(retval)
+
+  def extra_link_args(self):
+    """Returns a pre-processed list containing extra link arguments.
+    
+    Equivalent command line version:
+
+    .. code-block:: sh
+
+       $ PKG_CONFIG_PATH=<paths> pkg-config --libs-only-other <name>
+
+    """
+
+    status, stdout, stderr = self.__xcall__(['--libs-only-other'])
+
+    if status != 0:
+      raise RuntimeError("error querying --libs-only-other for package `%s': %s" % (self.package, stderr))
+
+    return stdout.strip().split()
+
+  def variable_names(self):
+    """Returns a list with all variable names know to this package
+    
+    Equivalent command line version:
+
+    .. code-block:: sh
+
+       $ PKG_CONFIG_PATH=<paths> pkg-config --print-variables <name>
+    
+    """
+
+    status, stdout, stderr = self.__xcall__(['--print-variables'])
+
+    if status != 0:
+      raise RuntimeError("error querying --print-variables for package `%s': %s" % (self.package, stderr))
+
+    return stdout.strip().split()
+
+  def variable(self, name):
+    """Returns a variable with a specific name (if it exists)
+    
+    Equivalent command line version:
+
+    .. code-block:: sh
+
+       $ PKG_CONFIG_PATH=<paths> pkg-config --variable=<variable-name> <name>
+
+    .. warning::
+
+       If a variable does not exist in a package, pkg-config does not signal an
+       error. Instead, it returns an empty string. So, do we.
+    """
+
+    status, stdout, stderr = self.__xcall__(['--variable=%s' % name])
+
+    if status != 0:
+      raise RuntimeError("error querying --variable=%s for package `%s': %s" % (name, self.package, stderr))
+
+    return stdout.strip()
+
+  def package_macros(self):
+    """Returns package availability and version number macros
+
+    This method returns a python list with 2 macros indicating package
+    availability and a version number, using standard GNU compatible names. For
+    example, if the package is named ``foo`` and its version is ``1.4``, this
+    command would return:
+
+    .. code-block:: sh
+
+       >>> foo = pkgconfig('foo')
+       >>> foo.package_macros()
+       [('HAVE_FOO', '1'), ('FOO_VERSION', '"1.4"')]
+
+    """
+    from re import sub
+    NAME = sub(r'[\.\-\s]', '_', self.name.upper())
+    return [('HAVE_' + NAME, '1'), (NAME + '_VERSION', '"%s"' % self.version)]
+
+__all__ = ['pkgconfig']

xbob/extension/test_pkgconfig.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Andre Anjos <andre.anjos@idiap.ch>
+# Wed 16 Oct 12:16:49 2013 
+
+"""Tests for pkgconfig
+"""
+
+import nose
+from .pkgconfig import pkgconfig
+
+test_package = 'zlib'
+
+def test_detect_ok():
+  pkg = pkgconfig(test_package)
+  nose.tools.eq_(pkg.name, test_package)
+  assert pkg.version
+  #print pkg.name, pkg.version
+
+@nose.tools.raises(RuntimeError)
+def test_detect_not_ok():
+  pkg = pkgconfig('foobarfoo')
+
+def test_include_directories():
+  pkg = pkgconfig(test_package)
+  obj = pkg.include_directories()
+  assert isinstance(obj, list)
+  assert obj
+  for k in obj:
+    assert k.find('-I') != 0 
+  #print obj
+
+def test_cflags_other():
+  pkg = pkgconfig('QtCore')
+  obj = pkg.cflags_other()
+  assert obj['define_macros']
+  assert isinstance(obj['define_macros'], list)
+  assert isinstance(obj['define_macros'][0], tuple)
+  assert isinstance(obj, dict)
+  #print obj
+
+def test_libraries():
+  pkg = pkgconfig(test_package)
+  obj = pkg.libraries()
+  assert isinstance(obj, list)
+  assert obj
+  for k in obj:
+    assert k.find('-l') != 0 
+  #print obj
+
+def test_library_directories():
+  pkg = pkgconfig(test_package)
+  obj = pkg.library_directories()
+  assert isinstance(obj, list)
+  assert obj
+  for k in obj:
+    assert k.find('-L') != 0
+  #print obj
+
+def test_extra_link_args():
+  pkg = pkgconfig(test_package)