utils.py 18.4 KB
Newer Older
1 2 3 4 5 6 7
#!/usr/bin/env python
# encoding: utf-8
# Andre Anjos <andre.dos.anjos@gmail.com>
# Fri 21 Mar 2014 10:37:40 CET

'''General utilities for building extensions'''

8 9 10 11 12
import os
import re
import sys
import glob
import platform
André Anjos's avatar
André Anjos committed
13
import pkg_resources
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
14
from . import DEFAULT_PREFIXES
15

16

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
17
def construct_search_paths(prefixes=None, subpaths=None, suffix=None):
18 19
  """Constructs a list of candidate paths to search for.

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
20 21 22 23 24
  The list of paths is constructed using the following order of priority:

  1. ``BOB_PREFIX_PATH`` environment variable, if set. ``BOB_PREFIX_PATH`` can
     contain several paths divided by :any:`os.pathsep`.
  2. The paths provided with the ``prefixes`` parameter.
25
  3. The current python executable prefix.
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
26
  4. The ``CONDA_PREFIX`` environment variable, if set.
27 28 29 30
  5. :any:`DEFAULT_PREFIXES`.

  Parameters
  ----------
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
31
  prefixes : [:obj:`str`], optional
32
      The list of paths to be added to the results.
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
33 34
  subpaths : [:obj:`str`], optional
      A list of subpaths to be appended to each path at the end. For
35 36
      example, if you specify ``['foo', 'bar']`` for this parameter, then
      ``os.path.join(paths[0], 'foo')``,
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
37 38 39 40 41
      ``os.path.join(paths[0], 'bar')``, and so on are added to the returned
      paths. Globs are accepted in this list and resolved using the function
      :py:func:`glob.glob`.
  suffix : :obj:`str`, optional
      ``suffix`` will be appended to all paths except ``prefixes``.
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56

  Returns
  -------
  paths : [str]
      A list of unique and existing paths to be used in your search.
  """
  search = []
  suffix = suffix or ''

  # Priority 1: the environment
  if 'BOB_PREFIX_PATH' in os.environ:
    paths = os.environ['BOB_PREFIX_PATH'].split(os.pathsep)
    search += [p + suffix for p in paths]

  # Priority 2: user passed paths
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
57 58
  if prefixes:
    search += prefixes
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73

  # Priority 3: the current system executable
  search.append(os.path.dirname(os.path.dirname(sys.executable)) + suffix)

  # Priority 4: the conda prefix
  conda_prefix = os.environ.get('CONDA_PREFIX')
  if conda_prefix:
    search.append(conda_prefix + suffix)

  # Priority 5: the default search prefixes
  search += [p + suffix for p in DEFAULT_PREFIXES]

  # Make unique to avoid searching twice
  search = uniq_paths(search)

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
74 75
  # Exhaustive combination of paths and subpaths
  if subpaths:
76 77
    subsearch = []
    for s in search:
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
78
      for p in subpaths:
79 80 81 82 83 84 85 86 87 88 89 90
        subsearch.append(os.path.join(s, p))
      subsearch.append(s)
    search = subsearch

  # Before we do a file-system check, filter out the un-existing paths
  tmp = []
  for k in search:
    tmp += glob.glob(k)
  search = tmp

  return search

91 92

def find_file(name, subpaths=None, prefixes=None):
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
93
  """Finds a generic file on the file system. Returns all occurrences.
94 95

  This method will find all occurrences of a given name on the file system and
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
96 97
  will return them to the user. It uses :any:`construct_search_paths` to
  construct the candidate folders that file may exist in.
98

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
99 100 101 102 103 104 105 106
  Parameters
  ----------
  name : str
      The name of the file. For example, ``gcc``.
  subpaths : [:obj:`str`], optional
      See :any:`construct_search_paths`
  subpaths : :obj:`str`, optional
      See :any:`construct_search_paths`
107

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
108 109 110 111 112
  Returns
  -------
  [str]
      A list of filenames that exist on the filesystem, matching your
      description.
113 114
  """

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
115
  search = construct_search_paths(prefixes=prefixes, subpaths=subpaths)
116 117 118 119

  retval = []
  for path in search:
    candidate = os.path.join(path, name)
120 121
    if os.path.exists(candidate):
      retval.append(candidate)
122 123 124

  return retval

125

126 127 128 129
def find_header(name, subpaths=None, prefixes=None):
  """Finds a header file on the file system. Returns all candidates.

  This method will find all occurrences of a given name on the file system and
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
130 131 132
  will return them to the user. It uses :any:`construct_search_paths` to
  construct the candidate folders that header may exist in accounting
  automatically for typical header folder names.
133

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
134 135 136 137 138 139 140 141
  Parameters
  ----------
  name : str
      The name of the header file.
  subpaths : [:obj:`str`], optional
      See :any:`construct_search_paths`
  subpaths : :obj:`str`, optional
      See :any:`construct_search_paths`
142

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
143 144 145 146 147
  Returns
  -------
  [str]
      A list of filenames that exist on the filesystem, matching your
      description.
148 149
  """

150
  headerpaths = []
151

152 153 154 155 156 157 158
  # arm-based system (e.g. raspberry pi 32 or 64-bit)
  if platform.machine().startswith('arm'):
    headerpaths += [os.path.join('include', 'arm-linux-gnueabihf')]

  # else, consider it intel compatible
  elif platform.architecture()[0] == '32bit':
    headerpaths += [os.path.join('include', 'i386-linux-gnu')]
159
  else:
160 161 162 163
    headerpaths += [os.path.join('include', 'x86_64-linux-gnu')]

  # Raspberry PI search directory (arch independent) + normal include
  headerpaths += ['include']
164

165 166
  # Exhaustive combination of paths and subpaths
  if subpaths:
167 168 169
    my_subpaths = []
    for hp in headerpaths:
      my_subpaths += [os.path.join(hp, k) for k in subpaths]
170
  else:
171
    my_subpaths = headerpaths
172 173 174

  return find_file(name, my_subpaths, prefixes)

175

176 177 178 179 180
def find_library(name, version=None, subpaths=None, prefixes=None,
    only_static=False):
  """Finds a library file on the file system. Returns all candidates.

  This method will find all occurrences of a given name on the file system and
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
181 182 183
  will return them to the user. It uses :any:`construct_search_paths` to
  construct the candidate folders that the library may exist in accounting
  automatically for typical library folder names.
184

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199
  Parameters
  ----------
  name : str
      The name of the module to be found. If you'd like to find libz.so, for
      example, specify ``"z"``. For libmath.so, specify ``"math"``.
  version : :obj:`str`, optional
      The version of the library we are searching for. If not specified, then
      look only for the default names, such as ``libz.so`` and the such.
  subpaths : [:obj:`str`], optional
      See :any:`construct_search_paths`
  subpaths : :obj:`str`, optional
      See :any:`construct_search_paths`
  only_static : :obj:`bool`, optional
      A boolean, indicating if we should try only to search for static versions
      of the libraries. If not set, any would do.
200

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
201 202 203 204 205
  Returns
  -------
  [str]
      A list of filenames that exist on the filesystem, matching your
      description.
206 207
  """

208 209 210 211 212
  libpaths = []

  # arm-based system (e.g. raspberry pi 32 or 64-bit)
  if platform.machine().startswith('arm'):
    libpaths += [os.path.join('lib', 'arm-linux-gnueabihf')]
213

214 215
  # else, consider it intel compatible
  elif platform.architecture()[0] == '32bit':
216 217 218 219 220 221 222 223 224 225
    libpaths += [
        os.path.join('lib', 'i386-linux-gnu'),
        os.path.join('lib32'),
        ]
  else:
    libpaths += [
        os.path.join('lib', 'x86_64-linux-gnu'),
        os.path.join('lib64'),
        ]

226 227
  libpaths += ['lib']

228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
  # Exhaustive combination of paths and subpaths
  if subpaths:
    my_subpaths = []
    for lp in libpaths:
      my_subpaths += [os.path.join(lp, k) for k in subpaths]
  else:
    my_subpaths = libpaths

  # Extensions to consider
  if only_static:
    extensions = ['.a']
  else:
    if sys.platform == 'darwin':
      extensions = ['.dylib', '.a']
    elif sys.platform == 'win32':
      extensions = ['.dll', '.a']
    else: # linux like
      extensions = ['.so', '.a']

  # The module names can be set with or without version number
  retval = []
  if version:
    for ext in extensions:
      if sys.platform == 'darwin': # version in the middle
        libname = 'lib' + name + '.' + version + ext
      else: # version at the end
        libname = 'lib' + name + ext + '.' + version

      retval += find_file(libname, my_subpaths, prefixes)

  for ext in extensions:
    libname = 'lib' + name + ext
    retval += find_file(libname, my_subpaths, prefixes)

  return retval

264 265 266 267
def find_executable(name, subpaths=None, prefixes=None):
  """Finds an executable on the file system. Returns all candidates.

  This method will find all occurrences of a given name on the file system and
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
268 269 270
  will return them to the user. It uses :any:`construct_search_paths` to
  construct the candidate folders that the executable may exist in accounting
  automatically for typical executable folder names.
271

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
272 273 274 275 276 277
  Parameters
  ----------
  name : str
      The name of the file. For example, ``gcc``.
  subpaths : [:obj:`str`], optional
      See :any:`construct_search_paths`
278
  prefixes : :obj:`str`, optional
Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
279
      See :any:`construct_search_paths`
280

Amir MOHAMMADI's avatar
Amir MOHAMMADI committed
281 282 283 284 285
  Returns
  -------
  [str]
      A list of filenames that exist on the filesystem, matching your
      description.
286 287
  """

288 289 290 291 292
  binpaths = []

  # arm-based system (e.g. raspberry pi 32 or 64-bit)
  if platform.machine().startswith('arm'):
    binpaths += [os.path.join('bin', 'arm-linux-gnueabihf')]
293

294 295
  # else, consider it intel compatible
  elif platform.architecture()[0] == '32bit':
296 297 298 299 300 301 302 303 304 305
    binpaths += [
        os.path.join('bin', 'i386-linux-gnu'),
        os.path.join('bin32'),
        ]
  else:
    binpaths += [
        os.path.join('bin', 'x86_64-linux-gnu'),
        os.path.join('bin64'),
        ]

306 307
  binpaths += ['bin']

308 309 310 311 312 313 314 315
  # Exhaustive combination of paths and subpaths
  if subpaths:
    my_subpaths = []
    for lp in binpaths:
      my_subpaths += [os.path.join(lp, k) for k in subpaths]
  else:
    my_subpaths = binpaths

316 317 318 319 320 321
  # if conda-build's BUILD_PREFIX is set, use it as it may contain build tools
  # which are not available on the host environment
  prefixes = prefixes if prefixes is not None else []
  if 'BUILD_PREFIX' in os.environ:
    prefixes += [os.environ['BUILD_PREFIX']]

322 323 324
  # The module names can be set with or without version number
  return find_file(name, my_subpaths, prefixes)

325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345
def uniq(seq, idfun=None):
  """Very fast, order preserving uniq function"""

  # order preserving
  if idfun is None:
      def idfun(x): return x
  seen = {}
  result = []
  for item in seq:
      marker = idfun(item)
      # in old Python versions:
      # if seen.has_key(marker)
      # but in new ones:
      if marker in seen: continue
      seen[marker] = 1
      result.append(item)
  return result

def uniq_paths(seq):
  """Uniq'fy a list of paths taking into consideration their real paths"""
  return uniq([os.path.realpath(k) for k in seq if os.path.exists(k)])
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369

def egrep(filename, expression):
  """Runs grep for a given expression on each line of the file

  Parameters:

  filename, str
    The name of the file to grep for the expression

  expression
    A regular expression, that will be initialized using :py:func:`re.compile`.

  Returns a list of re matches.
  """

  retval = []

  with open(filename, 'rt') as f:
    rexp = re.compile(expression)
    for line in f:
      p = rexp.match(line)
      if p: retval.append(p)

  return retval
370 371 372 373 374 375 376

def load_requirements(f=None):
  """Loads the contents of requirements.txt on the given path.

  Defaults to "./requirements.txt"
  """

377 378
  def readlines(f):
    retval = [str(k.strip()) for k in f]
379
    return [k for k in retval if k and k[0] not in ('#', '-')]
380

381 382 383 384 385 386 387 388
  # if f is None, use the default ('requirements.txt')
  if f is None:
    f = 'requirements.txt'
  if isinstance(f, str):
    f = open(f, 'rt')
  # read the contents
  return readlines(f)

389 390 391 392 393 394 395 396 397 398 399 400 401 402
def find_packages(directories=['bob']):
  """This function replaces the ``find_packages`` command from ``setuptools`` to search for packages only in the given directories.
  Using this function will increase the building speed, especially when you have (links to) deep non-code-related directory structures inside your package directory.
  The given ``directories`` should be a list of top-level sub-directories of your package, where package code can be found.
  By default, it uses ``'bob'`` as the only directory to search.
  """
  from setuptools import find_packages as _original
  if isinstance(directories, str):
    directories = [directories]
  packages = []
  for d in directories:
    packages += [d]
    packages += ["%s.%s" % (d, p) for p in _original(d)]
  return packages
403

404
def link_documentation(additional_packages = ['python', 'numpy'], requirements_file = "../requirements.txt", server = None):
405
  """Generates a list of documented packages on our documentation server for the packages read from the "requirements.txt" file and the given list of additional packages.
406 407 408 409

  Parameters:

  additional_packages : [str]
410 411
    A list of additional bob packages for which the documentation urls are added.
    By default, 'numpy' is added
412

413
  requirements_file : str or file-like
414
    The file (relative to the documentation directory), where to read the requirements from.
415
    If ``None``, it will be skipped.
416

417
  server : str or None
418
    The url to the server which provides the documentation.
419
    If ``None`` (the default), the ``BOB_DOCUMENTATION_SERVER`` environment variable is taken if existent.
420
    If neither ``server`` is specified, nor a ``BOB_DOCUMENTATION_SERVER`` environment variable is set, the default ``"http://www.idiap.ch/software/bob/docs/bob/%(name)s/%(version)s/"`` is used.
421 422

  """
423 424 425 426 427 428

  def smaller_than(v1, v2):
    """Compares scipy/numpy version numbers"""

    c1 = v1.split('.')
    c2 = v2.split('.')[:len(c1)] #clip to the compared version
429
    for i in range(len(c2)):
430 431 432 433 434 435 436 437
      n1 = c1[i]
      n2 = c2[i]
      try:
        n1 = int(n1)
        n2 = int(n2)
      except ValueError:
        n1 = str(n1)
        n2 = str(n2)
438
      if n1 < n2: return True
439
      if n1 > n2: return False
440
    return False
441 442


443 444
  if sys.version_info[0] <= 2:
    import urllib2 as urllib
445
    from urllib2 import HTTPError, URLError
446 447 448 449
  else:
    import urllib.request as urllib
    import urllib.error as error
    HTTPError = error.HTTPError
450
    URLError = error.URLError
451

452 453

  # collect packages are automatically included in the list of indexes
454
  packages = []
André Anjos's avatar
André Anjos committed
455
  version_re = re.compile(r'\s*[\<\>=]+\s*')
456
  if requirements_file is not None:
457 458 459
    if not isinstance(requirements_file, str) or \
        os.path.exists(requirements_file):
      requirements = load_requirements(requirements_file)
André Anjos's avatar
André Anjos committed
460
      packages += [version_re.split(k)[0] for k in requirements]
461 462
  packages += additional_packages

463

464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
  def _add_index(name, addr, packages=packages):
    """Helper to add a new doc index to the intersphinx catalog

    Parameters:

      name (str): Name of the package that will be added to the catalog
      addr (str): The URL (except the ``objects.inv`` file), that will be added

    """

    if name in packages:
      print ("Adding intersphinx source for `%s': %s" % (name, addr))
      mapping[name] = (addr, None)
      packages = [k for k in packages if k != name]


  def _add_numpy_index():
    """Helper to add the numpy manual"""

483 484
    try:
      import numpy
485 486 487
      ver = numpy.version.version
      if smaller_than(ver, '1.5.z'):
        ver = '.'.join(ver.split('.')[:-1]) + '.x'
488
      else:
489
        ver = '.'.join(ver.split('.')[:-1]) + '.0'
490
      _add_index('numpy', 'https://docs.scipy.org/doc/numpy-%s/' % ver)
491

492
    except ImportError:
493
      _add_index('numpy', 'https://docs.scipy.org/doc/numpy/')
494 495


496 497 498
  def _add_scipy_index():
    """Helper to add the scipy manual"""

499 500
    try:
      import scipy
501 502 503
      ver = scipy.version.version
      if smaller_than(ver, '0.9.0'):
        ver = '.'.join(ver.split('.')[:-1]) + '.x'
504
      else:
505
        ver = '.'.join(ver.split('.')[:-1]) + '.0'
506
      _add_index('scipy', 'https://docs.scipy.org/doc/scipy-%s/reference/' % ver)
507

508
    except ImportError:
509
      _add_index('scipy', 'https://docs.scipy.org/doc/scipy/reference/')
510 511


512 513 514 515 516 517 518 519 520
  def _add_click_index():
    """Helper to add the click manual"""

    import click
    major = click.__version__.split('.')[0]
    ver = major + '.x'
    _add_index('click', 'https://click.palletsprojects.com/en/%s/' % ver)


521 522 523
  mapping = {}

  # add indexes for common packages used in Bob
524
  _add_index('python', 'https://docs.python.org/%d.%d/' % sys.version_info[:2])
525 526
  _add_numpy_index()
  _add_scipy_index()
527 528
  _add_index('matplotlib', 'http://matplotlib.org/')
  _add_index('setuptools', 'https://setuptools.readthedocs.io/en/latest/')
529
  _add_index('six', 'https://six.readthedocs.io')
530 531
  _add_index('sqlalchemy', 'https://docs.sqlalchemy.org/en/latest/')
  _add_index('docopt', 'http://docopt.readthedocs.io/en/latest/')
532
  _add_index('scikit-learn', 'http://scikit-learn.org/stable/')
533 534
  _add_index('scikit-image', 'http://scikit-image.org/docs/dev/')
  _add_index('pillow', 'http://pillow.readthedocs.io/en/latest/')
535
  _add_click_index()
536
  _add_index('torch', 'https://pytorch.org/docs/')
André Anjos's avatar
André Anjos committed
537 538


539 540 541 542 543
  # get the server for the other packages
  if server is None:
    if "BOB_DOCUMENTATION_SERVER" in os.environ:
      server = os.environ["BOB_DOCUMENTATION_SERVER"]
    else:
544
      server = "http://www.idiap.ch/software/bob/docs/bob/%(name)s/%(version)s/|http://www.idiap.ch/software/bob/docs/bob/%(name)s/master/"
545

546 547 548 549
  # array support for BOB_DOCUMENTATION_SERVER
  # transforms "(file:///path/to/dir  https://example.com/dir| http://bla )"
  # into ["file:///path/to/dir", "https://example.com/dir", "http://bla"]
  # so, trim eventual parenthesis/white-spaces and splits by white space or |
André Anjos's avatar
André Anjos committed
550 551 552 553
  if server.strip():
    server = re.split(r'[|\s]+', server.strip('() '))
  else:
    server = []
554

André Anjos's avatar
André Anjos committed
555
  # check if the packages have documentation on the server
556
  for p in packages:
557 558
    if p in mapping: continue #do not add twice...

559 560
    for s in server:
      # generate URL
561 562 563 564
      package_name = p.split()[0]
      if s.count('%s') == 1: #old style
        url = s % package_name
      else: #use new style, with mapping, try to link against specific version
565
        try:
566
          version = 'v' + pkg_resources.require(package_name)[0].version
567 568
        except pkg_resources.DistributionNotFound:
          version = 'stable' #package is not a runtime dep, only referenced
569
        url = s % {'name': package_name, 'version': version}
570

571
      try:
572
        # otherwise, urlopen will fail
573 574
        if url.startswith('file://'):
          f = urllib.urlopen(urllib.Request(url + 'objects.inv'))
575
          url = url[7:] #intersphinx does not like file://
576 577
        else:
          f = urllib.urlopen(urllib.Request(url))
578

579
        # request url
580 581
        print("Found documentation for %s on %s; adding intersphinx source" % (p, url))
        mapping[p] = (url, None)
582 583 584 585 586
        break #inner loop, for server, as we found a candidate!

      except HTTPError as exc:
        if exc.code != 404:
          # url request failed with a something else than 404 Error
587
          print("Requesting URL %s returned error: %s" % (url, exc))
588 589 590
          # notice mapping is not updated here, as the URL does not exist

      except URLError as exc:
591 592 593 594 595
        print("Requesting URL %s did not succeed (maybe offline?). " \
            "The error was: %s" % (url, exc))

      except IOError as exc:
        print ("Path %s does not exist. The error was: %s" % (url, exc))
596

597
  return mapping