diff --git a/bob/devtools/scripts/create.py b/bob/devtools/scripts/create.py index 3751b37212d7096f27f575737837203b3f42d506..901ebe1d9157b444278d696f484dc826350fae54 100644 --- a/bob/devtools/scripts/create.py +++ b/bob/devtools/scripts/create.py @@ -31,7 +31,7 @@ Examples: \b $ cd bob.package.foo - $ bdt create -vv myenv + $ bdt dev create -vv myenv The above command assumes the directory `conda' exists on the current directory and that it contains a file called `meta.yaml' containing the recipe for the package you want to create a development environment for. @@ -40,12 +40,12 @@ Examples: 2. By default, we use the native python version of your conda installation as the default python version to use for the newly created environment. You may select a different one with `--python=X.Y': - $ bdt create -vv --python=3.6 myenv + $ bdt dev create -vv --python=3.6 myenv 3. By default, we use our own condarc and `conda_build_config.yaml` files that are used in creating packages for our CI/CD system. If you wish to use your own, specify them on the command line: - $ bdt create -vv --python=3.6 --config=config.yaml --condarc=~/.condarc myenv + $ bdt dev create -vv --python=3.6 --config=config.yaml --condarc=~/.condarc myenv Notice the condarc file **must** end in `condarc', or conda will complain. @@ -55,7 +55,7 @@ Examples: enable debug printing. Equivalent conda commands you can execute on the shell will be printed: - $ bdt create -vvv --dry-run myenv + $ bdt dev create -vvv --dry-run myenv 5. You can use the option `--pip-extras` to force the installation of extra @@ -65,7 +65,7 @@ Examples: explained in our Setup subsection of the Installation manual. To use this flag on the command-line, specify one pip-installable package each time: - $ bdt create -vvv --pip-extras=ipdb --pip-extras=mr.developer myenv + $ bdt dev create -vvv --pip-extras=ipdb --pip-extras=mr.developer myenv Using this option **adds** to what is available in the configuration file. So, if your configuration file already contains ``ipdb`` and you wish to diff --git a/bob/devtools/scripts/development.py b/bob/devtools/scripts/development.py new file mode 100644 index 0000000000000000000000000000000000000000..c7d5fbf0191c5302ba68da1f16832255a4bf7e9f --- /dev/null +++ b/bob/devtools/scripts/development.py @@ -0,0 +1,96 @@ +import click + +from click_plugins import with_plugins +from pkg_resources import iter_entry_points + + +@click.command(epilog="See bdt dev --help") +@click.argument( + "folders", + nargs=-1, + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +def install(folders): + """runs pip install -vvv --no-build-isolation --no-dependencies --editable <folder>""" + import os + import subprocess + + for folder in folders: + if not os.path.exists(os.path.join(folder, "setup.py")): + raise click.ClickException( + "Folder does not contain a setup.py: %s" % folder + ) + + # call pip + subprocess.check_call( + [ + "pip", + "install", + "-vvv", + "--no-build-isolation", + "--no-dependencies", + "--editable", + folder, + ] + ) + + +@click.command(epilog="See bdt dev --help") +@click.argument("names", nargs=-1) +@click.option("--use-https/--use-ssh", is_flag=True, default=False) +@click.option( + "-s", "--subfolder", default="", help="subfolder to checkout into" +) +@click.pass_context +def checkout(ctx, names, use_https, subfolder): + """git clones a Bob package.""" + import os + import subprocess + + # create the subfolder directory + if subfolder: + os.path.makedirs(subfolder, exist_ok=True) + + for name in names: + + # call git + # skip if the directory already exists + dest = name + if subfolder: + dest = os.path.join(subfolder, name) + + if not os.path.isdir(dest): + + url = f"git@gitlab.idiap.ch:bob/{name}.git" + if use_https: + url = f"https://gitlab.idiap.ch/bob/{name}.git" + + subprocess.check_call(["git", "clone", url, dest]) + + +@with_plugins(iter_entry_points("bdt.dev.cli")) +@click.group( + epilog="""Examples: + +\b +# develop an existing project +bdt dev checkout bob.bio.face +cd bob.bio.face +bdt dev create --python 3.9 bobbioface +bdt dev install . + +\b +# later on, checkout and develop more packages +bdt dev checkout --subfolder src bob.bio.base +bdt dev install src/bob.bio.base + +\b +# develop a new project +bdt dev new -vv bob/bob.newpackage "John Doe" "joe@example.com" +# edit the conda/meta.yaml and requirements.txt files to add your dependencies +bdt dev create --python 3.9 bobnewpackage +bdt install .""" +) +def dev(): + """Development scripts""" + pass diff --git a/bob/devtools/scripts/new.py b/bob/devtools/scripts/new.py index a0e26271280b1bab7692da7062ced929f4da9076..98c282518316e7a2e2181fa4685c25bdb019cce7 100644 --- a/bob/devtools/scripts/new.py +++ b/bob/devtools/scripts/new.py @@ -67,7 +67,7 @@ Examples: 1. Generates a new project for Bob: - $ bdt new -vv bob/bob.newpackage "John Doe" "joe@example.com" + $ bdt dev new -vv bob/bob.newpackage "John Doe" "joe@example.com" """ ) @click.argument("package") diff --git a/conda/meta.yaml b/conda/meta.yaml index 5aa391f3f4d57f620efe71c49bc92a77d5acd1c1..bb96f583505ced027b6c0a8bb9b1acd297d13bd6 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -68,16 +68,16 @@ test: - bdt --help - bdt dumpsphinx --help - bdt dumpsphinx https://docs.python.org/3/objects.inv > /dev/null - - bdt create --help + - bdt dev create --help - bdt build --help - bdt mirror --help - bdt rebuild --help - bdt test --help - bdt caupdate --help - - bdt new --help - - bdt new -t "New package" -o bob.foobar bob/bob.foobar "John Doe" "joe.doe@example.com" - - bdt new -t "New package" -l bsd -o bob.foobar2 bob/bob.foobar "John Doe" "joe.doe@example.com" - - bdt new -t "New package" -l bsd -o beat.foobar beat/beat.foobar "John Doe" "joe.doe@example.com" + - bdt dev new --help + - bdt dev new -t "New package" -o bob.foobar bob/bob.foobar "John Doe" "joe.doe@example.com" + - bdt dev new -t "New package" -l bsd -o bob.foobar2 bob/bob.foobar "John Doe" "joe.doe@example.com" + - bdt dev new -t "New package" -l bsd -o beat.foobar beat/beat.foobar "John Doe" "joe.doe@example.com" - bdt gitlab --help - bdt gitlab lasttag --help #- bdt gitlab lasttag -vv bob/bob.devtools diff --git a/doc/development.rst b/doc/development.rst index d2c8f4c3184114df4e5fe1c36347c1790a5cd32e..d5dc95bd306822a32fa31670ae14317f52784abb 100644 --- a/doc/development.rst +++ b/doc/development.rst @@ -6,36 +6,42 @@ Very often, developers are confronted with the need to clone package repositories locally and develop installation/build and runtime code. -It is recommended to create isolated environments to develop new projects using conda_ and zc.buildout_. -Tools implemented in ``bob.devtools`` helps automate this process for |project| packages. In the following we talk about how to checkout and build one or several packages from their git_ source and build proper isolated environments to develop them. Then we will describe how to create a new bob package from scratch and develop existing bob packages along side it. +It is recommended to create isolated environments using conda_ to develop new projects. +Tools implemented in ``bob.devtools`` helps automate this process for |project| packages. +In the following we talk about how to checkout and build one or several packages +from their git_ source and build proper isolated environments to develop them. +Then we will describe how to create a new bob package from scratch and develop +existing bob packages along side it. TLDR ==== -Suppose you want to checkout the package ``bob.blitz`` from source and start developing it locally. We will use the tools implemented in ``bob.devtools`` to create a proper developing environment to build and develop ``bob.blitz``. We assume you have ``bob.devtools`` installed on a conda environment named ``bdt`` (Refer to :ref:`bob.devtools.install` for detailed information.) +Suppose you want to checkout the package ``bob.blitz`` from source and start developing it locally. +We will use the tools implemented in ``bob.devtools`` to create a proper +developing environment to build and develop ``bob.blitz``. +We assume you have ``bob.devtools`` installed on a conda environment named +``bdt`` (Refer to :ref:`bob.devtools.install` for detailed information.) * Checkout the source of the package from git: .. code-block:: sh - $ git clone https://gitlab.idiap.ch/bob/bob.blitz + $ bdt dev checkout --use-ssh bob.blitz * Create a proper conda environment: .. code-block:: sh $ cd bob.blitz - $ conda activate bdt - $ bdt create -vv dev - $ conda deactivate + $ bdt dev create -vv dev $ conda activate dev -* Build the package using buildout: +* Build the package using pip: .. code-block:: sh - $ buildout - $ ./bin/python # you should use this python to run things from now on + $ bdt dev install . # calls pip with correct arguments + $ python for example: @@ -63,15 +69,7 @@ for example: .. code-block:: sh - $ ./bin/nosetests -sv - -.. note:: - - Sometimes when you are calling a function not interactively it is not acting normally. In that case import ``pkg_resources`` before importing your package. It is a known issue and we are working on it. - - .. code-block:: sh - - $ ./bin/python -c "import pkg_resources; import bob.blitz; print(bob.blitz)" + $ nosetests -sv * Some packages may come with a pre-commit_ config file (``.pre-commit-config.yaml``). Make sure to install pre-commit if the config file exists: @@ -87,18 +85,20 @@ for example: Local development of existing packages ====================================== -To develop existing |project| packages you need to checkout their source code and create a proper development environment using `buildout`. +To develop existing |project| packages you need to checkout their source code and install them in your environment. Checking out package sources ---------------------------- -|project| packages are developed through gitlab_. Various packages exist -in |project|'s gitlab_ instance. In the following we assume you want to install and build locally the ``bob.blitz`` package. In order to checkout a package, just use git_: +|project| packages are developed through gitlab_. Various packages exist in +|project|'s gitlab_ instance. In the following we assume you want to install and +build locally the ``bob.blitz`` package. In order to checkout a package, just +use git_: .. code-block:: sh - $ git clone https://gitlab.idiap.ch/bob/bob.blitz + $ bdt dev checkout --use-ssh bob.blitz Create an isolated conda environment @@ -110,27 +110,30 @@ Before proceeding, you need to make sure that you already have a conda_ environm .. code-block:: sh $ cd bob.blitz - $ conda activate bdt - $ bdt create -vv dev - $ conda deactivate + $ bdt dev create -vv dev $ conda activate dev -Now you have an isolated conda environment named `dev` with proper channels set. For more information about conda channels refer to `conda channel documentation`_. +Now you have an isolated conda environment named `dev` with proper channels set. +For more information about conda channels refer to `conda channel +documentation`_. -The `bdt create` command assumes a directory named `conda`, exists on the current directory. The `conda` directory contains a file named `meta.yaml`, that is the recipe required to create a development environment for the package you want to develop. +The `bdt dev create` command assumes a directory named `conda`, exists on the +current directory. The `conda` directory contains a file named `meta.yaml`, that +is the recipe required to create a development environment for the package you +want to develop. .. note:: When developing and testing new features, one often wishes to work against the very latest, *bleeding edge*, available set of changes on dependent packages. - `bdt create` command adds `Bob beta channels`_ to highest priority which creates an environment with the latest dependencies instead of the latest *stable* versions of each package. + `bdt dev create` command adds `Bob beta channels`_ to highest priority which creates an environment with the latest dependencies instead of the latest *stable* versions of each package. If you want to create your environment using *stable* channels, you can use this command instead: .. code-block:: sh - $ bdt create --stable -vv dev + $ bdt dev create --stable -vv dev To see which channels you are using run: @@ -147,72 +150,30 @@ The `bdt create` command assumes a directory named `conda`, exists on the curren environments which can be very different form each other. -Running buildout ----------------- - -The last step is to create a hooked-up environment so you can quickly test -local changes to your package w/o necessarily creating a conda-package. -zc.buildout_ takes care of it by modifying the load paths of scripts to find the correct -version of your package sources from the local checkout. It by default uses a file named `buildout.cfg`, in the package directory. For our example package it looks like: - -.. code-block:: ini +Installing the package +---------------------- - ; vim: set fileencoding=utf-8 : - ; Mon 08 Aug 2016 14:33:54 CEST - - [buildout] - parts = scripts - develop = . - eggs = bob.blitz - extensions = bob.buildout - newest = false - verbose = true - - [scripts] - recipe = bob.buildout:scripts - -To find our more information about different section of this file, refer to :ref:`bob.devtools.buildout`. - -Now you just need to run buildout: +The last step is to install the package: .. code-block:: sh $ cd bob.blitz #if that is not the case $ conda activate dev #if that is not the case - $ buildout - - - -After running, buildout creates a directory called ``bin`` on your local package checkout. Use -the applications living there to develop your package. For example, if you need -to run the test suite: + $ bdt dev install . +To run the test suite: .. code-block:: sh - $ ./bin/nosetests -sv + $ nosetests -sv or build the documentation: .. code-block:: sh - $./bin/sphinx-build -aEn doc sphinx # make sure it finishes without warnings. + $ sphinx-build -aEn doc sphinx # make sure it finishes without warnings. $ firefox sphinx/index.html # view the docs. -.. note:: - - `buildout` by default uses the file `buildout.cfg` but you can specify another file by using -c option. In fact for developing packages especially if they need to be developed along with other packages, another file, namely `develop.cfg` is used like following: - - .. code-block:: sh - - $ buildout -c develop.cfg - - -A python interpreter clone can be used to run interactive sessions: - -.. code-block:: sh - - $ ./bin/python You can see what is installed in your environment: @@ -233,54 +194,29 @@ And you can install new packages using conda: .. code-block:: sh $ cd <package> - $ conda activate bdt $ bdt local build -One important advantage of using conda_ and zc.buildout_ is that it does -**not** require administrator privileges for setting up any of the above. -Furthermore, you will be able to create distributable environments for each -project you have. This is a great way to release code for laboratory exercises -or for a particular publication that depends on |project|. +One important advantage of using conda_ is that it does **not** require +administrator privileges for setting up any of the above. Furthermore, you will +be able to create distributable environments for each project you have. This is +a great way to release code for laboratory exercises or for a particular +publication that depends on |project|. Developing multiple existing packages simultaneously ---------------------------------------------------- -It so happens that you want to develop several packages against each other for your project. Let's assume you want to develop ``bob.blitz`` and ``bob.extension`` simultaneously. ``bob.blitz`` is dependent on ``bob.devtools``. First we checkout package ``bob.blitz`` and build an isolated conda environment as explained in the previous section. Then edit `buildout.cfg` file (or `develop.cfg`) and add ``bob.extension`` to it as following: - - -.. code-block:: ini - - [buildout] - parts = scripts - - develop = src/bob.extension - . +It so happens that you want to develop several packages against each other for +your project. Let's assume you want to develop ``bob.blitz`` and +``bob.extension`` simultaneously. ``bob.blitz`` is dependent on +``bob.extension``. First we checkout package ``bob.blitz`` and build an isolated +conda environment as explained in the previous section. Then checkout and +install ``bob.extension`` as following: - eggs = bob.blitz - extensions = bob.buildout - mr.developer - - auto-checkout = * - - - ; options for bob.buildout extension - debug = true - verbose = true - newest = false - - [sources] - bob.extension = git https://gitlab.idiap.ch/bob/bob.extension - - [scripts] - recipe = bob.buildout:scripts - - -Now you can run `buildout` as usual. The ``bob.extension`` will be checked out on `src` folder on the root of your project. - -.. note:: +.. code-block:: sh - The flag `debug = true` is usually used when in development mode. + $ bdt dev checkout --use-ssh --subfolder src bob.extension + $ bdt dev install src/bob.extension .. _bob.devtools.create_package: @@ -288,98 +224,56 @@ Now you can run `buildout` as usual. The ``bob.extension`` will be checked out o Local development of a new package ================================== -In this section we explain how to create a new bob package from scratch and start developing it. Once again ``bob.devtools`` is here to help you. You need to activate your conda environment with ``bob.devtools`` installed in it. +In this section we explain how to create a new bob package from scratch and +start developing it. Once again ``bob.devtools`` is here to help you. You need +to activate your conda environment with ``bob.devtools`` installed in it. .. code-block:: sh - $ conda activate bdt - $ bdt new -vv bob/bob.project.awesome author_name author_email - -This command will create a new bob package named "bob.project.awesome" that includes the correct anatomy of a package. For more information about the functionality of each file check :ref:`bob.devtools.anatomy`. - -In the root of your project there is a file `buildout.cfg` used by `buildout` to build your package locally. It should look like: - -.. code-block:: ini - - [buildout] - parts = scripts - develop = . - eggs = bob.project.awesome - extensions = bob.buildout - newest = false - verbose = true - - [scripts] - recipe = bob.buildout:scripts - dependent-scripts = true + $ bdt dev new -vv bob/bob.project.awesome author_name author_email +This command will create a new bob package named "bob.project.awesome" that +includes the correct anatomy of a package. For more information about the +functionality of each file check :ref:`bob.devtools.anatomy`. -Now you have all the necessary tools available and you can make a development environment using `bdt create` command, run `buildout` in it and start developing your package. +Now you have all the necessary tools available and you can make a development +environment using `bdt dev create` command, run `bdt dev install` in it and +start developing your package. .. code-block:: sh $ cd bob.project.awesome - $ conda activate bdt - $ bdt create --stable -vv awesome-project #here we used the stable channels to make the conda environment. + $ bdt dev create --stable -vv awesome-project #here we used the stable channels to make the conda environment. $ conda activate awesome-project - $ buildout + $ bdt dev install . Developing existing bob packages along with your new package ------------------------------------------------------------ -Let's assume you need to develop two packages, ``bob.extension`` and ``bob.blitz``, as part of developing your new ``bob.project.awesome`` package. +Let's assume you need to develop two packages, ``bob.extension`` and +``bob.blitz``, as part of developing your new ``bob.project.awesome`` package. -You need to add these packages to the ``buildout.cfg`` file in the newly created folder. +You need to checkout and install these packages: -.. code-block:: ini - - [buildout] - parts = scripts - - develop = src/bob.extension - src/bob.blitz - . - - eggs = bob.extension - bob.blitz - bob.project.awesome - - extensions = bob.buildout - mr.developer - - auto-checkout = * - newest = false - verbose = true - - [scripts] - recipe = bob.buildout:scripts - dependent-scripts = true - - [sources] - bob.extension = git https://gitlab.idiap.ch/bob/bob.extension - bob.blitz = git https://gitlab.idiap.ch/bob/bob.blitz - ; or - ; bob.extension = git git@gitlab.idiap.ch:bob/bob.extension.git - ; bob.blitz = git git@gitlab.idiap.ch:bob/bob.blitz.git +.. code-block:: sh + $ bdt dev checkout --use-ssh --subfolder src bob.extension bob.blitz + $ bdt dev install src/bob.extension src/bob.blitz # the order of installing dependencies matters! -When you build your new package the dependent packages (in this example ``bob.extension`` and ``bob.blitz``) will be checked out on folder `src` in the root of your project. +When you build your new package, it is customary to checkout the dependent +packages (in this example ``bob.extension`` and ``bob.blitz``) in the `src` +folder in the root of your project. -As usual, first create an isolated conda environment using `bdt create` command. Some of bob packages need dependencies that might not be installed on your environment. You can find these dependencies by checking `conda/meta.yaml` of each package. Install the required packages and then run buildout as usual. For our example you need to do the following: +As usual, first create an isolated conda environment using `bdt dev create` command. +Some of bob packages need dependencies that might not be installed on your +environment. You can find these dependencies by checking `conda/meta.yaml` of +each package. Install the required packages and then run `bdt dev install`. For +our example you need to do the following: .. code-block:: sh $ conda install gcc_linux-64 gxx_linux-64 libblitz - $ buildout - -.. note:: - - Sometimes you may need some of bob packages available in your local `bin` directory without necessarily developing them. - - If you knew beforehand what are those packages, you can add them to "requirements/host" section of the `conda/meta.yaml` file and then create a conda environment using `bdt create`. Like this, those packages will be installed automatically. Otherwise, if you already have your conda environment, install them using `conda install` command. - - When done, add those packages to the `eggs` section in your `buildout.cfg` file and then run `buildout`. - + $ bdt dev install src/bob.extension src/bob.blitz # the order of installing dependencies matters! .. include:: links.rst diff --git a/doc/install.rst b/doc/install.rst index feac0ebfaad09f769939ce7c2dc9d906c7bcdfaa..61b3252e50e0150fd45f7345797a7c840643b2c8 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -17,7 +17,7 @@ channels: $ conda create -n bdt -c https://www.idiap.ch/software/bob/conda/label/beta -c https://www.idiap.ch/software/bob/conda bob.devtools If you use one of our supported Python versions on your base environment, you -may also install ``bdt`` on it: +may also install ``bdt`` on it (recommended): .. code-block:: sh @@ -30,8 +30,11 @@ installed, you can use these tools within the created environment like this: .. code-block:: sh - $ conda activate base #or bdt, depending where you installed it - (bdt) $ bdt --help + $ ln -s $(which bdt) ~/.local/bin + # make sure ~/.local/bin is in your $PATH + $ export PATH=$HOME/.local/bin:$PATH + # use bdt any time now no matter which conda env is activated + $ bdt --help .. _bob.devtools.install.setup: @@ -78,7 +81,7 @@ You may obtain these parameters from our internal page explaining the `WebDAV configuration`_. For security reasons, you should also set ``chmod 600`` to this file. -To increment your development environments created with ``bdt create`` using +To increment your development environments created with ``bdt dev create`` using pip-installable packages, create a section named ``create`` in the file ``~/.bdtrc`` with the following contents, e.g.: @@ -89,7 +92,7 @@ pip-installable packages, create a section named ``create`` in the file bob.buildout mr.developer -Then, by default, ``bdt create`` will automatically pip install ``ipdb`` and +Then, by default, ``bdt dev create`` will automatically pip install ``ipdb`` and ``mr.developer`` at environment creation time. You may reset this list to your liking. diff --git a/doc/templates.rst b/doc/templates.rst index 4c543a798e784165bca55986d8aa4f4cb72d0158..3990303df03badd47822e8c9d6cd0153d7886341 100644 --- a/doc/templates.rst +++ b/doc/templates.rst @@ -31,7 +31,7 @@ To run the test units on your package call: .. code-block:: sh - $ ./bin/nosetests -v + $ nosetests -v bob.example.library.test.test_reverse ... ok ---------------------------------------------------------------------- @@ -824,7 +824,7 @@ Buildout ======== The default buildout file ``buildout.cfg`` should buildout from the installed -distribution (use ``bdt create`` for that purpose) and **avoid mr.developer +distribution (use ``bdt dev create`` for that purpose) and **avoid mr.developer checkouts**. If you have one of those, move it to ``develop.cfg`` and create a new `buildout.cfg` which should be as simple as possible. The template project provided by this package takes care of this. diff --git a/setup.py b/setup.py index a05b9e19a18d145615d8b4150dc57e1dad5bcc49..f85badbc0bb9df2f2e9227067145b22b6f691201 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ setup( "local = bob.devtools.scripts.local:local", "gitlab = bob.devtools.scripts.gitlab:gitlab", "sphinx = bob.devtools.scripts.sphinx:sphinx", + "dev = bob.devtools.scripts.development:dev", ], "bdt.gitlab.cli": [ "badges = bob.devtools.scripts.badges:badges", @@ -106,6 +107,12 @@ setup( "upload = bob.devtools.scripts.dav:upload", "clean-betas = bob.devtools.scripts.dav:clean_betas", ], + "bdt.dev.cli": [ + "new = bob.devtools.scripts.new:new", + "create = bob.devtools.scripts.create:create", + "install = bob.devtools.scripts.development:install", + "checkout = bob.devtools.scripts.development:checkout", + ], }, classifiers=[ "Framework :: Bob",