diff --git a/release/README.md b/release/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3a0f4620a780d241b9f6ccabe21fff6f18710cb4 --- /dev/null +++ b/release/README.md @@ -0,0 +1,239 @@ +Here are some instructions to do a massive Bob release and scripts to help you +release Bob: + +Create a buildout configuration file that checks out all packages from +`bob.nightlies`. Here is an example: + + [buildout] + parts = scripts + eggs = bob.extension + bob.blitz + bob.core + ... + bob # don't forget bob + ... + bob.bio.video + bob.bio.vein + bob.db.voxforge + bob.pad.base + + + + extensions = bob.buildout + mr.developer + auto-checkout = * + develop = src/bob.extension + src/bob.blitz + src/bob.core + ... + src/bob.bio.video + src/bob.bio.vein + src/bob.db.voxforge + src/bob.pad.base + + + ; options for bob.buildout extension + debug = false + verbose = true + newest = false + + [sources] + bob.extension = git git@gitlab.idiap.ch:bob/bob.extension + bob.blitz = git git@gitlab.idiap.ch:bob/bob.blitz + bob.core = git git@gitlab.idiap.ch:bob/bob.core + ... + bob.bio.video = git git@gitlab.idiap.ch:bob/bob.bio.video + bob.bio.vein = git git@gitlab.idiap.ch:bob/bob.bio.vein + bob.db.voxforge = git git@gitlab.idiap.ch:bob/bob.db.voxforge + bob.pad.base = git git@gitlab.idiap.ch:bob/bob.pad.base + + [scripts] + recipe = bob.buildout:scripts + +List and order of packages come from bob.nightlies; in `core.txt` and +`extra.txt`. Don't forget to add `bob` after `core.txt`. + +Create a conda environment with `bob-devel`: + + $ conda create -n bob-devel-27 python=2 bob-devel + +You should not install any other package and run: + + $ buildout + $ bin/nosetests -sv src/* + +Make sure all the tests are passing. + +Go through all packages and write down what has changed since their last +tag and write it down into a file (`changelog.md` for example). Here is an +example: + + bob.extension (Minor) + + * !37 !38 !39 !41 : Improved documentation for a better development guide + * !42 : Improve the new version script in terms of functionality and documentation + + bob.blitz (Patch) + + * Maintenance release. + +**This is the most important part!!!!** Do not write this changelog easily. +You really have to dig through the history of all packages since the last +release. Write down what has changed by not copying and pasting. +Changelogs are read by humans and must be generated by humans. + +PROPERLY identify if a package needs a patch, minor, or a major release bump. +See http://semver.org for information. Ask the package's maintainer if you are +not sure. + +Then you can use `release.sh` and modify its release list depending on what you +have in your `changelog.md`. Some packages need a patch release and some need a +minor release and some a major release. + + Hold off `bob` for now. + Don't release private packages! + Read bin/bob_new_version.py --help + +Run that script. Release the first package. Wait for its CI to finish. Merge +its merge request on bob.conda. wait for that to finish in master of bob.conda. +Press `y` to go to the next package. + +Finally read `bob` s readme and release that too. + +You are done here no need to read the rest of this file. + +You may want to disable the runners for bob.conda so that you can run it once +everything is merged into master. To merge everything and cancel all pipelines +in bob.conda. Read here: +https://python-gitlab.readthedocs.io/en/stable/index.html + +Then: + + import gitlab + gl = gitlab.Gitlab.from_config() + bob_conda = gl.projects.search('bob.conda')[0] + + # get all merge requests and merge them + mrs = bob_conda.mergerequests.list(all=True, state='opened') + print(mrs) + for mr in mrs: + try: + mr.merge() + except Exception: + print('failed for ', mr.name) + pass + + # get all pipelines and merge them + pipelines = bob_conda.pipelines.list(all=True, scope='running') + print(len(pipelines)) + for p in pipelines: + try: + p.cancel() + except Exception: + print('failed for ', p) + pass + +This will merge all merge requests. But sometimes the branch is created but +there is no merge request for that. Merge those branches manually! and cancel +the pipelines again. + +Run the pipeline for master of bob.conda once and fix till every conda package +is released. + + +Release `bob`. Here is some code to get the changelog generated automatically +for bob: + + + import gitlab + import distutils.version + import re + + gl = gitlab.Gitlab.from_config() + + # bob's new requirementst.txt file: + path = '..../git/bob/requirements.txt' + + bob_group = gl.groups.search('bob')[0] + + def get_max_version(versions): + + try: + v = list(reversed(sorted([distutils.version.StrictVersion(k) + for k in versions]))) + final = [k for k in v if not k.prerelease] + if final: + return final[0] + return v[0] + except Exception: + v = list(reversed(sorted([distutils.version.LooseVersion(k) + for k in versions]))) + final = [k for k in v if not re.search(r'[a-z]', k.vstring)] + if final: + return final[0] + return v[0] + + pkgs = {} + for line in open(path): + pkg, version = line.split('==') + pkgs[pkg.strip()] = version.strip() + + # verify bob is pointing to the latest tags of core packages. + for pkg, version in pkgs.items(): + projects = bob_group.projects.search(pkg) + project = [p for p in projects if p.name == pkg][0] + tags = project.tags.list(all=True) + versions = [tag.name[1:] for tag in tags] + max_version = get_max_version(versions) + max_version = '{}.{}.{}'.format(*max_version.version) + if max_version != version: + print(pkg, versions) + +Generate the changelog: + + import gitlab + import distutils.version + import datetime + from collections import OrderedDict + gl = gitlab.Gitlab.from_config() + bob_group = gl.groups.search('bob')[0] + path = '.../git/bobs/bob/requirements.txt' + pkgs = OrderedDict() + for line in open(path): + pkg, version = line.split('==') + pkgs[pkg.strip()] = version.strip() + + # release date of last Bob release + 1 day + last_release = datetime.datetime(2017, 2, 10) + + def get_datetime_from_tag(tag): + return datetime.datetime.strptime(tag.commit.committed_date[:-6], '%Y-%m-%dT%H:%M:%S.%f') + + def sort_tags(tags): + return sorted(tags, key=lambda x: get_datetime_from_tag(x)) + + def get_tag_changelog(tag): + try: + return tag.release.description + except Exception: + return '' + + for pkg, version in pkgs.items(): + projects = bob_group.projects.search(pkg) + project = [p for p in projects if p.name == pkg][0] + tags = project.tags.list(all=True) + # sort tags by date + tags = filter(lambda x: get_datetime_from_tag(x) >= last_release, tags) + tags = sort_tags(tags) + print('* ' + pkg) + for tag in tags: + print(' * ' + tag.name) + for line in get_tag_changelog(tag).split('\r\n'): + line = line.strip() + line = line.replace('!', pkg+'!') + line = line.replace('#', pkg+'#') + if not line: + continue + print(' '*5 + '* ' + line) + +Put this awesome changelog in https://gitlab.idiap.ch/bob/bob/tags diff --git a/release/release.sh b/release/release.sh new file mode 100644 index 0000000000000000000000000000000000000000..94a93e47e824963a6b729eb6198f53c4bccd5e8e --- /dev/null +++ b/release/release.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +set -ex + +for name in \ +'bob.extension --minor' \ +'bob.blitz' \ +'bob.core' \ +'bob.io.base --minor' \ +'bob.math' \ +'bob.measure --minor' \ +'bob.io.image' \ +'bob.db.base --minor' \ +'bob.db.base' \ +'bob.io.video' \ +'bob.io.matlab' \ +'bob.io.audio' \ +'bob.sp' \ +'bob.ap' \ +'bob.ip.base' \ +'bob.ip.color' \ +'bob.ip.draw' \ +'bob.ip.gabor' \ +'bob.learn.activation' \ +'bob.learn.libsvm' \ +'bob.learn.linear' \ +'bob.learn.mlp' \ +'bob.learn.em' \ +'bob.learn.boosting' \ +'bob.db.iris --minor' \ +'bob.db.wine' \ +'bob.db.mnist --minor' \ +'bob.db.atnt' \ +'bob.ip.facedetect' \ +'bob.ip.optflow.hornschunck' \ +'bob.ip.optflow.liu' \ +'bob.ip.flandmark' \ +'gridtk' \ +'bob.ip.skincolorfilter' \ +'bob.ip.facelandmarks' \ +'bob.ip.dlib' \ +'bob.db.arface' \ +'bob.db.asvspoof' \ +'bob.db.asvspoof2017' \ +'bob.db.atvskeystroke' \ +'bob.db.avspoof' \ +'bob.db.banca' \ +'bob.db.biosecure' \ +'bob.db.biosecurid.face' \ +'bob.db.casme2' \ +'bob.db.caspeal' \ +'bob.db.cohface' \ +'bob.db.frgc' \ +'bob.db.gbu' \ +'bob.db.hci_tagging' \ +'bob.db.ijba' \ +'bob.db.kboc16' \ +'bob.db.lfw' \ +'bob.db.livdet2013' \ +'bob.db.mobio' \ +'bob.db.msu_mfsd_mod' \ +'bob.db.multipie' \ +'bob.db.nist_sre12' \ +'bob.db.putvein' \ +'bob.db.replay' \ +'bob.db.replaymobile' \ +'bob.db.scface' \ +'bob.db.utfvp' \ +'bob.db.voicepa' \ +'bob.db.xm2vts' \ +'bob.db.youtube' \ +'bob.db.pericrosseye' \ +'bob.db.cuhk_cufs' \ +'bob.bio.base' \ +'bob.bio.gmm' \ +'bob.bio.face --minor' \ +'bob.bio.spear --minor' \ +'bob.bio.video --minor' \ +'bob.db.voxforge' \ +'bob.pad.base' \ +; do + stringarray=(${name}) + pkg=${stringarray[0]} + patch=${stringarray[1]} + cd src/${pkg} + git co master + git pull --rebase origin master + ../../bin/bob_new_version.py ${patch} + while true; do + read -p "Do you wish to continue?" yn + case $yn in + [Yy]* ) break;; + [Nn]* ) exit;; + * ) echo "Please answer yes or no.";; + esac + done + cd ../.. +done