diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index 4cf8d24c02290d5bcac5dd4544fe7f8b28e78b91..bf1ed405dd1d641cc1e6308ddc47f3bcd0ab1c76 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -290,7 +290,8 @@ def parse_dependencies(recipe_dir, config):
+ remove_pins(recipe["requirements"].get("host", []))
+ recipe["requirements"].get("run", [])
+ recipe.get("test", {}).get("requires", [])
- + ["bob.buildout", "mr.developer", "ipdb"]
+ + ["bob.buildout"] #required for basic bootstrap of most recipes
+ + ["ipython"] #for ipdb
# Also add anaconda compilers to make sure source installed packages are
# compiled properly
+ ["clangxx_osx-64" if platform.system() == "Darwin" else "gxx_linux-64"]
diff --git a/bob/devtools/config.py b/bob/devtools/config.py
new file mode 100755
index 0000000000000000000000000000000000000000..84282d3d282b30588337e83b962ec63f3f346304
--- /dev/null
+++ b/bob/devtools/config.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+"""Reads and treats configuration files"""
+
+
+import os
+import configparser
+
+
+def read_config():
+ """Reads and resolves configuration files, returns a dictionary with read
+ values"""
+
+ data = configparser.ConfigParser()
+ cfg = os.path.expanduser("~/.bdtrc")
+ if os.path.exists(cfg):
+ data.read(cfg)
+ return data
diff --git a/bob/devtools/dav.py b/bob/devtools/dav.py
index 3503258741718f9507c257f6be8e54392ad38c8e..7d30f14dd848c679bae7c010b7dbee31d583b515 100644
--- a/bob/devtools/dav.py
+++ b/bob/devtools/dav.py
@@ -10,6 +10,7 @@ from distutils.version import StrictVersion
import dateutil.parser
from .deploy import _setup_webdav_client
+from .config import read_config
from .log import echo_normal
from .log import echo_warning
from .log import get_logger
@@ -21,41 +22,31 @@ def _get_config():
"""Returns a dictionary with server parameters, or ask them to the user"""
# tries to figure if we can authenticate using a configuration file
- cfgs = ["~/.bdtrc"]
- cfgs = [os.path.expanduser(k) for k in cfgs]
- for k in cfgs:
- if os.path.exists(k):
- data = configparser.ConfigParser()
- data.read(k)
- if (
- "webdav" not in data
- or "server" not in data["webdav"]
+ data = read_config()
+
+ # this does some sort of validation for the "webdav" data...
+ if "webdav" in data:
+ if ("server" not in data["webdav"]
or "username" not in data["webdav"]
or "password" not in data["webdav"]
- ):
- assert KeyError, (
- "The file %s should contain a single "
- '"webdav" section with 3 variables defined inside: '
- '"server", "username", "password".' % (k,)
- )
- return data["webdav"]
-
- # ask the user for the information, cache credentials for future use
- retval = dict()
- retval["server"] = input("The base address of the server: ")
- retval["username"] = input("Username: ")
- retval["password"] = input("Password: ")
-
- # record file for the user
- data = configparser.ConfigParser()
- data["webdav"] = retval
- with open(cfgs[0], "w") as f:
- logger.warn('Recorded "%s" configuration file for next queries')
- data.write(f)
- os.chmod(cfgs[0], 0o600)
- logger.warn('Changed mode of "%s" to be read-only to you')
-
- return retval
+ ):
+ raise KeyError(
+ f"If the configuration file {k} contains a \"webdav\" " \
+ f"section, it should contain 3 variables defined inside: " \
+ f'"server", "username", "password".'
+ )
+ else:
+ # ask the user for the information, in case nothing available
+ logger.warn("Requesting server information for webDAV operation. " \
+ "(To create a configuration file, and avoid these, follow " \
+ "the Setup subsection at our Installation manual.)")
+ webdav_data = dict()
+ webdav_data["server"] = input("The base address of the server: ")
+ webdav_data["username"] = input("Username: ")
+ webdav_data["password"] = input("Password: ")
+ data["webdav"] = webdav_data
+
+ return data["webdav"]
def setup_webdav_client(private):
diff --git a/bob/devtools/scripts/create.py b/bob/devtools/scripts/create.py
index 7981287ba30e21bff07b09649f98b3f97f32a4ee..61d3b6f18af9f7be16cad35f0577167fecfb6c01 100644
--- a/bob/devtools/scripts/create.py
+++ b/bob/devtools/scripts/create.py
@@ -3,10 +3,12 @@
import os
import sys
+import subprocess
import click
import yaml
+from ..config import read_config
from ..bootstrap import set_environment
from ..build import conda_create
from ..build import make_conda_config
@@ -23,6 +25,14 @@ from . import bdt
logger = get_logger(__name__)
+def _uniq(seq):
+ """Fast order preserving uniq() function for Python lists"""
+
+ seen = set()
+ seen_add = seen.add
+ return [x for x in seq if not (x in seen or seen_add(x))]
+
+
@click.command(
epilog="""
Examples:
@@ -55,8 +65,23 @@ Examples:
enable debug printing. Equivalent conda commands you can execute on the
shell will be printed:
-
$ bdt create -vvv --dry-run myenv
+
+
+ 5. You can use the option `--pip-extras` to force the installation of extra
+ Python packages that are useful in your development environment. By default
+ we do not install anything, but you may configure this via this flag, or
+ through the configuration file option `create/pip_extras` to do so, as
+ explained in our Setup subsection of the Installation manual. To use this
+ flag on the command-line, specify one pip-installable package each time:
+
+ $ bdt create -vvv --pip-extras=ipdb --pip-extras=mr.developer myenv
+
+ Using this option **adds** to what is available in the configuration file.
+ So, if your configuration file already contains ``ipdb`` and you wish to
+ install ``mr.developer`` as a plus, then just specify
+ ``--pip-extras=mr.developer``.
+
"""
)
@click.argument("name")
@@ -143,6 +168,15 @@ Examples:
"(combine with the verbosity flags - e.g. ``-vvv``) to enable "
"printing to help you understand what will be done",
)
+@click.option(
+ "-x",
+ "--pip-extras",
+ multiple=True,
+ default=[],
+ help="Using pip, installs this additional list of dependencies to "
+ "created environments. Pip installation happens after the base conda "
+ "install (defaults, if any, are listed in ~/.bdtrc)",
+)
@verbosity_option()
@bdt.raise_on_error
def create(
@@ -159,6 +193,7 @@ def create(
private,
stable,
dry_run,
+ pip_extras,
):
"""Creates a development environment for a recipe.
@@ -231,4 +266,18 @@ def create(
# when creating a local development environment, remove the always_yes option
del condarc_options["always_yes"]
conda_create(conda, name, overwrite, condarc_options, deps, dry_run, use_local)
- echo_normal('Execute on your shell: "conda activate %s"' % name)
+
+ # part 2: pip-install everything listed in pip-extras
+ # mix-in stuff from ~/.bdtrc and command-line
+ config = read_config()
+ pip_extras_config = []
+ if "create" in config:
+ pip_extras_config = config["create"].get("pip_extras", "").split()
+ pip_extras = _uniq(pip_extras_config + list(pip_extras))
+ logger.info("Pip-installing: %s", pip_extras)
+
+ cmd = [conda, "run", "--live-stream", "--name", name, "pip", "install"]
+ cmd += pip_extras
+ subprocess.run(cmd, check=True, bufsize=1)
+
+ echo_normal(f">>> Execute on your shell: \"conda activate {name}\"")
diff --git a/bob/devtools/scripts/dav.py b/bob/devtools/scripts/dav.py
index 69d0d92bb769445926a98db933eb0ef5590efed3..26f161d9bdebc7298397edd4a5164b25385c235b 100644
--- a/bob/devtools/scripts/dav.py
+++ b/bob/devtools/scripts/dav.py
@@ -25,8 +25,10 @@ logger = get_logger(__name__)
def dav():
"""Commands for reading/listing/renaming/copying content to a WebDAV server
- Commands defined here may require a username and a password to operate
- properly.
+ Commands defined here require a server, username and a password to operate
+ properly. These values will be asked to you everytime you use a subcommand
+ in this group. To avoid repetitive asking, create a configuration file as
+ indicated in the :doc:`bob.devtools.install.setup` subsection of the manual.
"""
pass
diff --git a/doc/install.rst b/doc/install.rst
index d2da4d9456875c48e66318555b45355340093244..23bbec3dedb92d9fc5caf0dd442de42a9aff973d 100644
--- a/doc/install.rst
+++ b/doc/install.rst
@@ -34,6 +34,7 @@ installed, you can use these tools within the created environment like this:
(bdt) $ bdt --help
+.. _bob.devtools.install.setup:
Setup
=====
@@ -74,7 +75,21 @@ that server inside the file ``~/.bdtrc``. Here is a skeleton:
password = password
You may obtain these parameters from our internal page explaining the `WebDAV
-configuration`_. You shoul also set ``chmod 600`` to this file for obvious
-security reasons.
+configuration`_. For security reasons, you should also set ``chmod 600`` to
+this file.
+
+To increment your development environments created with ``bdt create`` using
+pip-installable packages, create a section named ``create`` in the file
+``~/.bdtrc`` with the following contents, e.g.:
+
+.. code-block:: ini
+
+ [create]
+ pip_extras = ipdb
+ mr.developer
+
+Then, by default, ``bdt create`` will automatically pip install ``ipdb`` and
+``mr.developer`` at environment creation time. You may reset this list to your
+liking.
.. include:: links.rst