Add a series of bdt dev commands to help in local development

Fixes #86

bob/devtools/scripts/create.py

@@ -31,7 +31,7 @@ Examples:
 
 \b
      $ cd bob.package.foo
-     $ bdt create -vv myenv
+     $ bdt dev create -vv myenv
 
      The above command assumes the directory `conda' exists on the current directory and that it contains a file called `meta.yaml' containing the recipe for the package you want to create a development environment for.
 
@@ -40,12 +40,12 @@ Examples:
 
   2. By default, we use the native python version of your conda installation as the default python version to use for the newly created environment. You may select a different one with `--python=X.Y':
 
-     $ bdt create -vv --python=3.6 myenv
+     $ bdt dev create -vv --python=3.6 myenv
 
 
   3. By default, we use our own condarc and `conda_build_config.yaml` files that are used in creating packages for our CI/CD system. If you wish to use your own, specify them on the command line:
 
-     $ bdt create -vv --python=3.6 --config=config.yaml --condarc=~/.condarc myenv
+     $ bdt dev create -vv --python=3.6 --config=config.yaml --condarc=~/.condarc myenv
 
      Notice the condarc file **must** end in `condarc', or conda will complain.
 
@@ -55,7 +55,7 @@ Examples:
   enable debug printing.  Equivalent conda commands you can execute on the
   shell will be printed:
 
-     $ bdt create -vvv --dry-run myenv
+     $ bdt dev create -vvv --dry-run myenv
 
 
   5. You can use the option `--pip-extras` to force the installation of extra
@@ -65,7 +65,7 @@ Examples:
   explained in our Setup subsection of the Installation manual.  To use this
   flag on the command-line, specify one pip-installable package each time:
 
-     $ bdt create -vvv --pip-extras=ipdb --pip-extras=mr.developer myenv
+     $ bdt dev create -vvv --pip-extras=ipdb --pip-extras=mr.developer myenv
 
      Using this option **adds** to what is available in the configuration file.
      So, if your configuration file already contains ``ipdb`` and you wish to

bob/devtools/scripts/development.py

+import click
+
+from click_plugins import with_plugins
+from pkg_resources import iter_entry_points
+
+
+@click.command(epilog="See bdt dev --help")
+@click.argument(
+    "folders",
+    nargs=-1,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
+)
+def install(folders):
+    """runs pip install -vvv --no-build-isolation --no-dependencies --editable <folder>"""
+    import os
+    import subprocess
+
+    for folder in folders:
+        if not os.path.exists(os.path.join(folder, "setup.py")):
+            raise click.ClickException(
+                "Folder does not contain a setup.py: %s" % folder
+            )
+
+        # call pip
+        subprocess.check_call(
+            [
+                "pip",
+                "install",
+                "-vvv",
+                "--no-build-isolation",
+                "--no-dependencies",
+                "--editable",
+                folder,
+            ]
+        )
+
+
+@click.command(epilog="See bdt dev --help")
+@click.argument("names", nargs=-1)
+@click.option("--use-https/--use-ssh", is_flag=True, default=False)
+@click.option(
+    "-s", "--subfolder", default="", help="subfolder to checkout into"
+)
+@click.pass_context
+def checkout(ctx, names, use_https, subfolder):
+    """git clones a Bob package."""
+    import os
+    import subprocess
+
+    # create the subfolder directory
+    if subfolder:
+        os.path.makedirs(subfolder, exist_ok=True)
+
+    for name in names:
+
+        # call git
+        # skip if the directory already exists
+        dest = name
+        if subfolder:
+            dest = os.path.join(subfolder, name)
+
+        if not os.path.isdir(dest):
+
+            url = f"git@gitlab.idiap.ch:bob/{name}.git"
+            if use_https:
+                url = f"https://gitlab.idiap.ch/bob/{name}.git"
+
+            subprocess.check_call(["git", "clone", url, dest])
+
+
+@with_plugins(iter_entry_points("bdt.dev.cli"))
+@click.group(
+    epilog="""Examples:
+
+\b
+# develop an existing project
+bdt dev checkout bob.bio.face
+cd bob.bio.face
+bdt dev create --python 3.9 bobbioface
+bdt dev install .
+
+\b
+# later on, checkout and develop more packages
+bdt dev checkout --subfolder src bob.bio.base
+bdt dev install src/bob.bio.base
+
+\b
+# develop a new project
+bdt dev new -vv bob/bob.newpackage "John Doe" "joe@example.com"
+# edit the conda/meta.yaml and requirements.txt files to add your dependencies
+bdt dev create --python 3.9 bobnewpackage
+bdt install ."""
+)
+def dev():
+    """Development scripts"""
+    pass

bob/devtools/scripts/new.py

@@ -67,7 +67,7 @@ Examples:
 
   1. Generates a new project for Bob:
 
-     $ bdt new -vv bob/bob.newpackage "John Doe" "joe@example.com"
+     $ bdt dev new -vv bob/bob.newpackage "John Doe" "joe@example.com"
 """
 )
 @click.argument("package")

conda/meta.yaml

@@ -68,16 +68,16 @@ test:
     - bdt --help
     - bdt dumpsphinx --help
     - bdt dumpsphinx https://docs.python.org/3/objects.inv > /dev/null
-    - bdt create --help
+    - bdt dev create --help
     - bdt build --help
     - bdt mirror --help
     - bdt rebuild --help
     - bdt test --help
     - bdt caupdate --help
-    - bdt new --help
-    - bdt new -t "New package" -o bob.foobar bob/bob.foobar "John Doe" "joe.doe@example.com"
-    - bdt new -t "New package" -l bsd -o bob.foobar2 bob/bob.foobar "John Doe" "joe.doe@example.com"
-    - bdt new -t "New package" -l bsd -o beat.foobar beat/beat.foobar "John Doe" "joe.doe@example.com"
+    - bdt dev new --help
+    - bdt dev new -t "New package" -o bob.foobar bob/bob.foobar "John Doe" "joe.doe@example.com"
+    - bdt dev new -t "New package" -l bsd -o bob.foobar2 bob/bob.foobar "John Doe" "joe.doe@example.com"
+    - bdt dev new -t "New package" -l bsd -o beat.foobar beat/beat.foobar "John Doe" "joe.doe@example.com"
     - bdt gitlab --help
     - bdt gitlab lasttag --help
       #- bdt gitlab lasttag -vv bob/bob.devtools

doc/development.rst

@@ -6,36 +6,42 @@
 
 Very often, developers are confronted with the need to
 clone package repositories locally and develop installation/build and runtime code.
-It is recommended to create isolated environments to develop new projects using conda_ and zc.buildout_.
-Tools implemented in ``bob.devtools`` helps automate this process for |project| packages. In the following we talk about how to checkout and build one or several packages from their git_ source and build proper isolated environments to develop them. Then we will describe how to create a new bob package from scratch and develop existing bob packages along side it.
+It is recommended to create isolated environments using conda_ to develop new projects.
+Tools implemented in ``bob.devtools`` helps automate this process for |project| packages.
+In the following we talk about how to checkout and build one or several packages
+from their git_ source and build proper isolated environments to develop them.
+Then we will describe how to create a new bob package from scratch and develop
+existing bob packages along side it.
 
 TLDR
 ====
 
-Suppose you want to checkout the package ``bob.blitz`` from source and start developing it locally. We will use the tools implemented in ``bob.devtools`` to create a proper developing environment to build and develop ``bob.blitz``. We assume you have ``bob.devtools`` installed on a conda environment named ``bdt`` (Refer to :ref:`bob.devtools.install` for detailed information.)
+Suppose you want to checkout the package ``bob.blitz`` from source and start developing it locally.
+We will use the tools implemented in ``bob.devtools`` to create a proper
+developing environment to build and develop ``bob.blitz``.
+We assume you have ``bob.devtools`` installed on a conda environment named
+``bdt`` (Refer to :ref:`bob.devtools.install` for detailed information.)
 
 * Checkout the source of the package from git:
 
 .. code-block:: sh
 
-   $ git clone https://gitlab.idiap.ch/bob/bob.blitz
+   $ bdt dev checkout --use-ssh bob.blitz
 
 * Create a proper conda environment:
 
 .. code-block:: sh
 
    $ cd bob.blitz
-   $ conda activate bdt
-   $ bdt create -vv dev
-   $ conda deactivate
+   $ bdt dev create -vv dev
    $ conda activate dev
 
-* Build the package using buildout:
+* Build the package using pip:
 
 .. code-block:: sh
 
-   $ buildout
-   $ ./bin/python  # you should use this python to run things from now on
+   $ bdt dev install . # calls pip with correct arguments
+   $ python
 
 for example:
 
@@ -63,15 +69,7 @@ for example:
 
 .. code-block:: sh
 
-   $ ./bin/nosetests -sv
-
-.. note::
-
-    Sometimes when you are calling a function not interactively it is not acting normally. In that case import ``pkg_resources`` before importing your package. It is a known issue and we are working on it.
-
-    .. code-block:: sh
-
-        $ ./bin/python -c "import pkg_resources; import bob.blitz; print(bob.blitz)"
+   $ nosetests -sv
 
 * Some packages may come with a pre-commit_ config file (``.pre-commit-config.yaml``).
   Make sure to install pre-commit if the config file exists:
@@ -87,18 +85,20 @@ for example:
 
 Local development of existing packages
 ======================================
-To develop existing |project| packages you need to checkout their source code and create a proper development environment using `buildout`.
+To develop existing |project| packages you need to checkout their source code and install them in your environment.
 
 
 Checking out package sources
 ----------------------------
-|project| packages are developed through gitlab_. Various packages exist
-in |project|'s gitlab_ instance. In the following we assume you want to install and build locally the ``bob.blitz`` package. In order to checkout a package, just use git_:
+|project| packages are developed through gitlab_. Various packages exist in
+|project|'s gitlab_ instance. In the following we assume you want to install and
+build locally the ``bob.blitz`` package. In order to checkout a package, just
+use git_:
 
 
 .. code-block:: sh
 
-   $ git clone https://gitlab.idiap.ch/bob/bob.blitz
+   $ bdt dev checkout --use-ssh bob.blitz
 
 
 Create an isolated conda environment
@@ -110,27 +110,30 @@ Before proceeding, you need to make sure that you already have a conda_ environm
 .. code-block:: sh
 
    $ cd bob.blitz
-   $ conda activate bdt
-   $ bdt create -vv dev
-   $ conda deactivate
+   $ bdt dev create -vv dev
    $ conda activate dev
 
 
-Now you have an isolated conda environment named `dev` with proper channels set. For more information about conda channels refer to `conda channel documentation`_.
+Now you have an isolated conda environment named `dev` with proper channels set.
+For more information about conda channels refer to `conda channel
+documentation`_.
 
-The `bdt create` command assumes a directory named `conda`, exists on the current directory. The `conda` directory contains a file named `meta.yaml`, that is the recipe required to create a development environment for the package you want to develop.
+The `bdt dev create` command assumes a directory named `conda`, exists on the
+current directory. The `conda` directory contains a file named `meta.yaml`, that
+is the recipe required to create a development environment for the package you
+want to develop.
 
 .. note::
 
     When developing and testing new features, one often wishes to work against the very latest, *bleeding edge*, available set of changes on dependent packages.
 
-    `bdt create` command adds `Bob beta channels`_ to highest priority which creates an environment with the latest dependencies instead of the latest *stable* versions of each package.
+    `bdt dev create` command adds `Bob beta channels`_ to highest priority which creates an environment with the latest dependencies instead of the latest *stable* versions of each package.
 
     If you want to create your environment using *stable* channels, you can use this command instead:
 
       .. code-block:: sh
 
-        $ bdt create --stable -vv dev
+        $ bdt dev create --stable -vv dev
 
     To see which channels you are using run:
 
@@ -147,72 +150,30 @@ The `bdt create` command assumes a directory named `conda`, exists on the curren
     environments which can be very different form each other.
 
 
-Running buildout
-----------------
-
-The last step is to create a hooked-up environment so you can quickly test
-local changes to your package w/o necessarily creating a conda-package.
-zc.buildout_ takes care of it by modifying the load paths of scripts to find the correct
-version of your package sources from the local checkout. It by default uses a file named `buildout.cfg`, in the package directory. For our example package it looks like:
-
-.. code-block:: ini
+Installing the package
+----------------------
 
-  ; vim: set fileencoding=utf-8 :
-  ; Mon 08 Aug 2016 14:33:54 CEST
-
-  [buildout]
-  parts = scripts
-  develop = .
-  eggs = bob.blitz
-  extensions = bob.buildout
-  newest = false
-  verbose = true
-
-  [scripts]
-  recipe = bob.buildout:scripts
-
-To find our more information about different section of this file, refer to :ref:`bob.devtools.buildout`.
-
-Now you just need to run buildout:
+The last step is to install the package:
 
 .. code-block:: sh
 
    $ cd bob.blitz #if that is not the case
    $ conda activate dev #if that is not the case
-   $ buildout
-
-
-
-After running, buildout creates a directory called ``bin`` on your local package checkout. Use
-the applications living there to develop your package. For example, if you need
-to run the test suite:
+   $ bdt dev install .
 
+To run the test suite:
 
 .. code-block:: sh
 
-   $ ./bin/nosetests -sv
+   $ nosetests -sv
 
 or build the documentation:
 
 .. code-block:: sh
 
-    $./bin/sphinx-build -aEn doc sphinx  # make sure it finishes without warnings.
+    $ sphinx-build -aEn doc sphinx  # make sure it finishes without warnings.
     $ firefox sphinx/index.html  # view the docs.
 
-.. note::
-
-    `buildout` by default uses the file `buildout.cfg` but you can specify another file by using -c option. In fact for developing packages especially if they need to be developed along with other packages, another file, namely `develop.cfg` is used like following:
-
-    .. code-block:: sh
-
-       $ buildout -c develop.cfg
-
-
-A python interpreter clone can be used to run interactive sessions:
-
-.. code-block:: sh
-
-   $ ./bin/python
 
 You can see what is installed in your environment:
 
@@ -233,54 +194,29 @@ And you can install new packages using conda:
     .. code-block:: sh
 
        $ cd <package>
-       $ conda activate bdt
        $ bdt local build
 
 
-One important advantage of using conda_ and zc.buildout_ is that it does
-**not** require administrator privileges for setting up any of the above.
-Furthermore, you will be able to create distributable environments for each
-project you have. This is a great way to release code for laboratory exercises
-or for a particular publication that depends on |project|.
+One important advantage of using conda_ is that it does **not** require
+administrator privileges for setting up any of the above. Furthermore, you will
+be able to create distributable environments for each project you have. This is
+a great way to release code for laboratory exercises or for a particular
+publication that depends on |project|.
 
 Developing multiple existing packages simultaneously
 ----------------------------------------------------
-It so happens that you want to develop several packages against each other for your project. Let's assume you want to develop ``bob.blitz`` and ``bob.extension`` simultaneously. ``bob.blitz`` is dependent on ``bob.devtools``. First we checkout package ``bob.blitz`` and build an isolated conda environment as explained in the previous section. Then edit `buildout.cfg` file (or `develop.cfg`) and add ``bob.extension`` to it as following:
-
-
-.. code-block:: ini
-
-    [buildout]
-    parts = scripts
-
-    develop = src/bob.extension
-              .
+It so happens that you want to develop several packages against each other for
+your project. Let's assume you want to develop ``bob.blitz`` and
+``bob.extension`` simultaneously. ``bob.blitz`` is dependent on
+``bob.extension``. First we checkout package ``bob.blitz`` and build an isolated
+conda environment as explained in the previous section. Then checkout and
+install ``bob.extension`` as following:
 
-    eggs = bob.blitz
 
-    extensions = bob.buildout
-                 mr.developer
-
-    auto-checkout = *
-
-
-    ; options for bob.buildout extension
-    debug = true
-    verbose = true
-    newest = false
-
-    [sources]
-    bob.extension = git https://gitlab.idiap.ch/bob/bob.extension
-
-    [scripts]
-    recipe = bob.buildout:scripts
-
-
-Now you can run `buildout` as usual. The ``bob.extension`` will be checked out on `src` folder on the root of your project.
-
-.. note::
+.. code-block:: sh
 
-  The flag `debug = true` is usually used when in development mode.
+    $ bdt dev checkout --use-ssh --subfolder src bob.extension
+    $ bdt dev install src/bob.extension
 
 
 .. _bob.devtools.create_package:
@@ -288,98 +224,56 @@ Now you can run `buildout` as usual. The ``bob.extension`` will be checked out o
 Local development of a new package
 ==================================
 
-In this section we explain how to create a new bob package from scratch and start developing it. Once again ``bob.devtools`` is here to help you. You need to activate your conda environment with ``bob.devtools`` installed in it.
+In this section we explain how to create a new bob package from scratch and
+start developing it. Once again ``bob.devtools`` is here to help you. You need
+to activate your conda environment with ``bob.devtools`` installed in it.
 
 .. code-block:: sh
 
-    $ conda activate bdt
-    $ bdt new -vv bob/bob.project.awesome author_name author_email
-
-This command will create a new bob package named "bob.project.awesome" that includes the correct anatomy of a package. For more information about the functionality of each file check :ref:`bob.devtools.anatomy`.
-
-In the root of your project there is a file `buildout.cfg` used by `buildout` to build your package locally. It should look like:
-
-.. code-block:: ini
-
-    [buildout]
-    parts = scripts
-    develop = .
-    eggs = bob.project.awesome
-    extensions = bob.buildout
-    newest = false
-    verbose = true
-
-    [scripts]
-    recipe = bob.buildout:scripts
-    dependent-scripts = true
+    $ bdt dev new -vv bob/bob.project.awesome author_name author_email
 
+This command will create a new bob package named "bob.project.awesome" that
+includes the correct anatomy of a package. For more information about the
+functionality of each file check :ref:`bob.devtools.anatomy`.
 
-Now you have all the necessary tools available and you can make a development environment using `bdt create` command, run `buildout` in it and start developing your package.
+Now you have all the necessary tools available and you can make a development
+environment using `bdt dev create` command, run `bdt dev install` in it and
+start developing your package.
 
 .. code-block:: sh
 
     $ cd bob.project.awesome
-    $ conda activate bdt
-    $ bdt create --stable -vv awesome-project  #here we used the stable channels to make the conda environment.
+    $ bdt dev create --stable -vv awesome-project  #here we used the stable channels to make the conda environment.
     $ conda activate awesome-project
-    $ buildout
+    $ bdt dev install .
 
 
 Developing existing bob packages along with your new package
 ------------------------------------------------------------
 
-Let's assume you need to develop two packages, ``bob.extension`` and ``bob.blitz``, as part of developing your new ``bob.project.awesome`` package.
+Let's assume you need to develop two packages, ``bob.extension`` and
+``bob.blitz``, as part of developing your new ``bob.project.awesome`` package.
 
-You need to add these packages to the ``buildout.cfg`` file in the newly created folder.
+You need to checkout and install these packages:
 
-.. code-block:: ini
-
-    [buildout]
-    parts = scripts
-
-    develop = src/bob.extension
-              src/bob.blitz
-              .
-
-    eggs = bob.extension
-           bob.blitz
-           bob.project.awesome
-
-    extensions = bob.buildout
-                 mr.developer
-
-    auto-checkout = *
-    newest = false
-    verbose = true
-
-    [scripts]
-    recipe = bob.buildout:scripts
-    dependent-scripts = true
-
-    [sources]
-    bob.extension = git https://gitlab.idiap.ch/bob/bob.extension
-    bob.blitz = git https://gitlab.idiap.ch/bob/bob.blitz
-    ; or
-    ; bob.extension = git git@gitlab.idiap.ch:bob/bob.extension.git
-    ; bob.blitz = git git@gitlab.idiap.ch:bob/bob.blitz.git
+.. code-block:: sh
 
+    $ bdt dev checkout --use-ssh --subfolder src bob.extension bob.blitz
+    $ bdt dev install src/bob.extension src/bob.blitz  # the order of installing dependencies matter!
 
-When you build your new package the dependent packages (in this example ``bob.extension`` and ``bob.blitz``) will be checked out on folder `src` in the root of your project.
+When you build your new package, it is customary to checkout the dependent
+packages (in this example ``bob.extension`` and ``bob.blitz``) in the `src`
+folder in the root of your project.
 
-As usual, first create an isolated conda environment using `bdt create` command. Some of bob packages need dependencies that might not be installed on your environment. You can find these dependencies by checking `conda/meta.yaml` of each package. Install the required packages and then run buildout as usual. For our example you need to do the following:
+As usual, first create an isolated conda environment using `bdt dev create` command.
+Some of bob packages need dependencies that might not be installed on your
+environment. You can find these dependencies by checking `conda/meta.yaml` of
+each package. Install the required packages and then run `bdt dev install`. For
+our example you need to do the following:
 
 .. code-block:: sh
 
     $ conda install gcc_linux-64 gxx_linux-64 libblitz
-    $ buildout
-
-.. note::
-
-    Sometimes you may need some of bob packages available in your local `bin` directory without necessarily developing them.
-
-    If you knew beforehand what are those packages, you can add them to "requirements/host" section of the `conda/meta.yaml` file and then create a conda environment using `bdt create`. Like this, those packages will be installed automatically. Otherwise, if you already have your conda environment, install them using `conda install` command.
-
-    When done, add those packages to the `eggs` section in your `buildout.cfg` file and then run `buildout`.
-
+    $ bdt dev install src/bob.extension src/bob.blitz  # the order of installing dependencies matter!
 
 .. include:: links.rst

doc/install.rst

@@ -17,7 +17,7 @@ channels:
    $ conda create -n bdt -c https://www.idiap.ch/software/bob/conda/label/beta -c https://www.idiap.ch/software/bob/conda bob.devtools
 
 If you use one of our supported Python versions on your base environment, you
-may also install ``bdt`` on it:
+may also install ``bdt`` on it (recommended):
 
 .. code-block:: sh
 
@@ -30,8 +30,11 @@ installed, you can use these tools within the created environment like this:
 
 .. code-block:: sh
 
-   $ conda activate base  #or bdt, depending where you installed it
-   (bdt) $ bdt --help
+   $ ln -s $(which bdt) ~/.local/bin
+   # make sure ~/.local/bin is in your $PATH
+   $ export PATH=$HOME/.local/bin:$PATH
+   # use bdt any time now no matter which conda env is activated
+   $ bdt --help
 
 
 .. _bob.devtools.install.setup:
@@ -78,7 +81,7 @@ You may obtain these parameters from our internal page explaining the `WebDAV
 configuration`_.  For security reasons, you should also set ``chmod 600`` to
 this file.
 
-To increment your development environments created with ``bdt create`` using
+To increment your development environments created with ``bdt dev create`` using
 pip-installable packages, create a section named ``create`` in the file
 ``~/.bdtrc`` with the following contents, e.g.:
 
@@ -89,7 +92,7 @@ pip-installable packages, create a section named ``create`` in the file
                 bob.buildout
                 mr.developer
 
-Then, by default, ``bdt create`` will automatically pip install ``ipdb`` and
+Then, by default, ``bdt dev create`` will automatically pip install ``ipdb`` and
 ``mr.developer`` at environment creation time.  You may reset this list to your
 liking.
 

doc/templates.rst

@@ -824,7 +824,7 @@ Buildout
 ========
 
 The default buildout file ``buildout.cfg`` should buildout from the installed
-distribution (use ``bdt create`` for that purpose) and **avoid mr.developer
+distribution (use ``bdt dev create`` for that purpose) and **avoid mr.developer
 checkouts**.  If you have one of those, move it to ``develop.cfg`` and create a
 new `buildout.cfg` which should be as simple as possible.  The template project
 provided by this package takes care of this.

setup.py

@@ -62,6 +62,7 @@ setup(
             "local = bob.devtools.scripts.local:local",
             "gitlab = bob.devtools.scripts.gitlab:gitlab",
             "sphinx = bob.devtools.scripts.sphinx:sphinx",
+            "dev = bob.devtools.scripts.development:dev",
         ],
         "bdt.gitlab.cli": [
             "badges = bob.devtools.scripts.badges:badges",
@@ -106,6 +107,12 @@ setup(
             "upload = bob.devtools.scripts.dav:upload",
             "clean-betas = bob.devtools.scripts.dav:clean_betas",
         ],
+        "bdt.dev.cli": [
+            "new = bob.devtools.scripts.new:new",
+            "create = bob.devtools.scripts.create:create",
+            "install = bob.devtools.scripts.development:install",
+            "checkout = bob.devtools.scripts.development:checkout",
+        ],
     },
     classifiers=[
         "Framework :: Bob",