Commit 6e24f0ce authored by André Anjos's avatar André Anjos 💬

Merge branch 'update-ci' into 'master'

Update CI and documentation linking

See merge request !272
parents 945a6c3b 7a90346b
Pipeline #27613 failed with stages
in 19 minutes and 32 seconds
# This build file uses template features from YAML so it is generic enough for
# any Bob project. Don't modify it unless you know what you're doing.
# Definition of global variables (all stages)
variables:
CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
DOCKER_REGISTRY: docker.idiap.ch
TEST_PREFIX_PATH_FILE: 'test_prefix_$CI_JOB_ID.txt'
BOOTSTRAP: "https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/bootstrap.py"
# Definition of our build pipeline order
# This build does not deploy to PyPI
stages:
- build
- deploy
# Build targets
.build_template: &build_job
build_linux_36:
stage: build
before_script:
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci master #installs ci support scripts
- ./_ci/before_build.sh
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN docker.idiap.ch
variables:
PYTHON_VERSION: "3.6"
TEST_PREFIX_PATH_FILE: 'test_prefix_$CI_JOB_ID.txt'
tags:
- docker-build
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- export BEAT_TEST_PREFIX=`mktemp -d --tmpdir=/var/tmp beat_test_prefix.XXXXXXXXX`
- echo $BEAT_TEST_PREFIX > $TEST_PREFIX_PATH_FILE
- ./scripts/before_build.sh
script:
- ./scripts/build.sh
after_script:
- ./scripts/after_build.sh
- ./_ci/after_build.sh
- conda env create -f dev.yml
- conda activate beat-web
- buildout
- python -c "from beat.core.test.utils import pull_docker_test_images as f; f()"
- export COVERAGE_FILE=.coverage.django
- ./bin/coverage run --source=${CI_PROJECT_NAME} ./bin/django test --settings=beat.web.settings.ci -v 2
- export BEAT_CMDLINE_TEST_PLATFORM="django://beat.web.settings.ci"
- export COVERAGE_FILE=.coverage.cmdline
- export NOSE_WITH_COVERAGE=1
- export NOSE_COVER_PACKAGE=beat.web
- ./bin/coverage run --source=beat.cmdline ./bin/nosetests -sv beat.cmdline
- unset COVERAGE_FILE
- ./bin/coverage combine .coverage.django .coverage.cmdline
- ./bin/coverage report
- ./bin/sphinx-apidoc --separate -d 2 --output=doc/api ${CI_PROJECT_NAMESPACE} beat/web/*/migrations beat/web/*/tests
- ./bin/sphinx-build doc/api sphinx/api
- ./bin/sphinx-build doc/admin sphinx/admin
- ./bin/sphinx-build doc/user sphinx/user
- conda deactivate
- export BEAT_TEST_PREFIX=`cat $TEST_PREFIX_PATH_FILE`
- rm -rf ${BEAT_TEST_PREFIX} ${TEST_PREFIX_PATH_FILE}
- conda activate base
- bdt ci clean -vv
artifacts:
expire_in: 1 week
paths:
- sphinx
cache: &build_caches
key: "linux-cache"
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
.build_linux_template: &linux_build_job
<<: *build_job
tags:
- docker-build
cache:
<<: *build_caches
key: "linux-cache"
build_linux_36:
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.6"
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/*.zip
- sphinx
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/deploy.sh
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- bdt ci deploy -vv
- bdt ci clean -vv
dependencies:
- build_linux_36
cache:
<<: *build_caches
tags:
- deployer
- docker
deploy_beta:
......
This diff is collapsed.
......@@ -28,7 +28,7 @@
from django.contrib.auth.models import User
from ...libraries.models import Library
from ...common.testutils import tearDownModule
from ...common.testutils import tearDownModule # noqa test runner will call it
from ..models import Algorithm
......@@ -38,18 +38,25 @@ from .core import AlgorithmsBaseTestCase
class DependencyTests(AlgorithmsBaseTestCase):
def setUp(self):
super(DependencyTests, self).setUp()
user = User.objects.create_user('johndoe', 'johndoe@test.org', '1234')
user = User.objects.get(username="johndoe")
algorithm_declaration = """{
"description": "some description",
"schema_version": 1,
"language": "python",
"splittable": false,
"groups": [
{
"name": "channel1",
"inputs": {
"in_data": {
"type": "johndoe/integer/1"
}
},
"outputs": {
"out_data": {
"type": "johndoe/integer/1"
}
}
}
],
......@@ -81,44 +88,42 @@ class DependencyTests(AlgorithmsBaseTestCase):
"""
(algorithm, errors) = Algorithm.objects.create_algorithm(
author=user,
name='algorithm1',
name="algorithm1",
declaration=algorithm_declaration,
code=algorithm_code,
)
assert algorithm, errors
self.assertIsNotNone(algorithm, errors)
(library, errors) = Library.objects.create_library(
author=user,
name='library1',
name="library1",
declaration=library_declaration,
code=library_code,
)
assert library, errors
self.assertIsNotNone(library, errors)
def test_add_dependency(self):
algorithm1 = Algorithm.objects.get(name="algorithm1")
library1 = Library.objects.get(name='library1')
library1 = Library.objects.get(name="library1")
decl = algorithm1.declaration
decl.setdefault('uses', {})
decl['uses']['dep1'] = library1.fullname()
decl.setdefault("uses", {})
decl["uses"]["dep1"] = library1.fullname()
algorithm1.declaration = decl
algorithm1.save()
self.assertEqual(algorithm1.referenced_libraries.count(), 1)
def test_update_dependency(self):
algorithm1 = Algorithm.objects.get(name="algorithm1")
library1 = Library.objects.get(name='library1')
library1 = Library.objects.get(name="library1")
decl = algorithm1.declaration
decl.setdefault('uses', {})
decl['uses']['dep1'] = library1.fullname()
decl['uses']['dep2'] = library1.fullname()
decl.setdefault("uses", {})
decl["uses"]["dep1"] = library1.fullname()
decl["uses"]["dep2"] = library1.fullname()
algorithm1.declaration = decl
algorithm1.save()
......@@ -130,7 +135,7 @@ class DependencyTests(AlgorithmsBaseTestCase):
self.assertEqual(algorithm1.referenced_libraries.count(), 1)
decl = algorithm1.declaration
decl['uses'] = {}
decl["uses"] = {}
algorithm1.declaration = decl
algorithm1.save()
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -25,25 +25,24 @@
# #
###############################################################################
from ...common.testutils import BaseTestCase as APITestCase
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from ...common.testutils import BaseTestCase as APITestCase
from ...common.testutils import tearDownModule # noqa test runner will call it
class CancelAllExperimentsAPI(APITestCase):
class CancelAllExperimentsAPI(APITestCase):
def setUp(self):
self.url = reverse('backend:cancel-experiments')
self.url = reverse("backend:cancel-experiments")
def test_no_access_for_anonymous_user(self):
response = self.client.get(self.url)
self.checkResponse(response, 302) #redirects to login page
self.checkResponse(response, 302) # redirects to login page
def test_no_access_for_non_superuser(self):
User.objects.create_user('johndoe', 'johndoe@test.org', '1234')
self.client.login(username='johndoe', password='1234')
PASSWORD = "1234"
User.objects.create_user("johndoe", "johndoe@test.org", PASSWORD)
self.client.login(username="johndoe", password=PASSWORD)
response = self.client.get(self.url)
self.checkResponse(response, 403)
......@@ -26,7 +26,6 @@
###############################################################################
import os
import sys
import time
import shutil
import tempfile
......@@ -34,43 +33,36 @@ import tempfile
from django.core import management
from django.test import TestCase
from ...common.testutils import tearDownModule # noqa test runner will call it
from ..utils import cleanup_cache
class CacheCleanUp(TestCase):
def setUp(self):
self.cache = tempfile.mkdtemp(prefix='beat_')
self.cache = tempfile.mkdtemp(prefix="beat_")
def tearDown(self):
shutil.rmtree(self.cache)
def touch(self, f, times=None):
"""Replicates the `touch' command-line utility"""
with open(f, 'a'): os.utime(f, times)
with open(f, "a"):
os.utime(f, times)
def J(self, *args):
return os.path.join(*((self.cache,) + args))
def prepare_cleanup_full(self):
# creates a temporary directory structure
os.makedirs(self.J('a', 'b', 'c'))
os.makedirs(self.J('a', 'c', 'd'))
os.makedirs(self.J('a', 'c', 'e'))
self.touch(self.J('a', 'b', 'c', 'd.json'))
self.touch(self.J('a', 'c', 'd', 'e.json'))
os.makedirs(self.J("a", "b", "c"))
os.makedirs(self.J("a", "c", "d"))
os.makedirs(self.J("a", "c", "e"))
self.touch(self.J("a", "b", "c", "d.json"))
self.touch(self.J("a", "c", "d", "e.json"))
def check_cleanup_full(self):
assert not os.listdir(self.cache)
self.assertIsNotNone(os.listdir(self.cache))
def test_cache_cleanup_full(self):
......@@ -78,34 +70,31 @@ class CacheCleanUp(TestCase):
cleanup_cache(self.cache, delete=True)
self.check_cleanup_full()
def test_cmd_cleanup_full(self):
self.prepare_cleanup_full()
management.call_command('cleanup_cache', path=self.cache,
verbosity=0, delete=True)
management.call_command(
"cleanup_cache", path=self.cache, verbosity=0, delete=True
)
self.check_cleanup_full()
def prepare_cleanup_aged(self):
two_min_ago = time.time() - 60*2
two_min_ago = time.time() - 60 * 2
# creates a temporary directory structure
os.makedirs(self.J('a', 'b', 'c'))
os.makedirs(self.J('a', 'c', 'd'))
os.makedirs(self.J('a', 'c', 'e'))
self.touch(self.J('a', 'b', 'c', 'd.json'), (two_min_ago, two_min_ago))
self.touch(self.J('a', 'c', 'd', 'e.json'))
os.makedirs(self.J("a", "b", "c"))
os.makedirs(self.J("a", "c", "d"))
os.makedirs(self.J("a", "c", "e"))
self.touch(self.J("a", "b", "c", "d.json"), (two_min_ago, two_min_ago))
self.touch(self.J("a", "c", "d", "e.json"))
def check_cleanup_aged(self):
assert os.path.exists(self.J('a', 'c', 'd', 'e.json'))
assert not os.path.exists(self.J('a', 'b', 'c'))
assert not os.path.exists(self.J('a', 'b', 'c', 'd.json'))
assert not os.path.exists(self.J('a', 'b', 'e'))
self.assertTrue(os.path.exists(self.J("a", "c", "d", "e.json")))
self.assertFalse(os.path.exists(self.J("a", "b", "c")))
self.assertFalse(os.path.exists(self.J("a", "b", "c", "d.json")))
self.assertFalse(os.path.exists(self.J("a", "b", "e")))
def test_cache_cleanup_aged(self):
......@@ -113,37 +102,34 @@ class CacheCleanUp(TestCase):
cleanup_cache(self.cache, age_in_minutes=2, delete=True)
self.check_cleanup_aged()
def test_cmd_cleanup_aged(self):
self.prepare_cleanup_aged()
management.call_command('cleanup_cache', path=self.cache,
verbosity=0, olderthan=2, delete=True)
management.call_command(
"cleanup_cache", path=self.cache, verbosity=0, olderthan=2, delete=True
)
self.check_cleanup_aged()
def prepare_cleanup_lock(self):
two_min_ago = time.time() - 60*2
ten_min_ago = time.time() - 60*10
two_min_ago = time.time() - 60 * 2
ten_min_ago = time.time() - 60 * 10
# creates a temporary directory structure
os.makedirs(self.J('a', 'b', 'c'))
os.makedirs(self.J('a', 'c', 'd'))
os.makedirs(self.J('a', 'c', 'e'))
self.touch(self.J('a', 'b', 'c', 'd.json'), (two_min_ago, two_min_ago))
self.touch(self.J('a', 'c', 'd', 'e.json'), (ten_min_ago, ten_min_ago))
self.touch(self.J('a', 'c', 'd', 'e.lock')) #create a lock
os.makedirs(self.J("a", "b", "c"))
os.makedirs(self.J("a", "c", "d"))
os.makedirs(self.J("a", "c", "e"))
self.touch(self.J("a", "b", "c", "d.json"), (two_min_ago, two_min_ago))
self.touch(self.J("a", "c", "d", "e.json"), (ten_min_ago, ten_min_ago))
self.touch(self.J("a", "c", "d", "e.lock")) # create a lock
def check_cleanup_lock(self):
assert os.path.exists(self.J('a', 'c', 'd', 'e.json'))
assert not os.path.exists(self.J('a', 'b', 'c'))
assert not os.path.exists(self.J('a', 'b', 'c', 'd.json'))
assert not os.path.exists(self.J('a', 'b', 'e'))
self.assertTrue(os.path.exists(self.J("a", "c", "d", "e.json")))
self.assertFalse(os.path.exists(self.J("a", "b", "c")))
self.assertFalse(os.path.exists(self.J("a", "b", "c", "d.json")))
self.assertFalse(os.path.exists(self.J("a", "b", "e")))
def test_cache_cleanup_lock(self):
......@@ -151,10 +137,10 @@ class CacheCleanUp(TestCase):
cleanup_cache(self.cache, delete=True)
self.check_cleanup_lock()
def test_cmd_cleanup_lock(self):
self.prepare_cleanup_lock()
management.call_command('cleanup_cache', path=self.cache,
verbosity=0, delete=True)
management.call_command(
"cleanup_cache", path=self.cache, verbosity=0, delete=True
)
self.check_cleanup_lock()
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -25,8 +25,7 @@
# #
###############################################################################
import simplejson as json
from datetime import datetime
import json
from django.contrib.auth.models import User
from django.conf import settings
......@@ -37,39 +36,42 @@ from .models import Database
from ..dataformats.models import DataFormat
from ..common.testutils import BaseTestCase
from ..common.testutils import tearDownModule # noqa test runner will call it
TEST_PWD = "1234"
class DatabaseAPIBase(BaseTestCase):
DATABASE = {
"root_folder": "/path/to/root/folder",
"protocols": [
{
"name": "protocol1",
"template": "test",
"sets": [
{
"name": "set1",
"template": "set",
"view": "dummy",
"outputs": {
"out": settings.SYSTEM_ACCOUNT + "/float/1"
}
}
]
}
]
"protocols": [
{
"name": "protocol1",
"template": "test",
"sets": [
{
"name": "set1",
"template": "set",
"view": "dummy",
"outputs": {"out": settings.SYSTEM_ACCOUNT + "/float/1"},
}
],
}
],
}
def setUp(self):
# Users
self.system_user = User.objects.create_user(settings.SYSTEM_ACCOUNT,
settings.SYSTEM_ACCOUNT + '@test.org', '1234')
self.system_user = User.objects.create_user(
settings.SYSTEM_ACCOUNT, settings.SYSTEM_ACCOUNT + "@test.org", TEST_PWD
)
self.system_user.is_staff = True
self.system_user.save()
user = User.objects.create_user('jackdoe', 'jackdoe@test.org', '1234')
User.objects.create_user('johndoe', 'johndoe@test.org', '1234')
User.objects.create_user("jackdoe", "jackdoe@test.org", TEST_PWD)
User.objects.create_user("johndoe", "johndoe@test.org", TEST_PWD)
self.db_name = 'test_db'
self.db_name = "test_db"
def tearDown(self):
pass
......@@ -79,47 +81,46 @@ class DatabaseCreationAPI(DatabaseAPIBase):
def setUp(self):
super(DatabaseCreationAPI, self).setUp()
self.url = reverse('api_databases:all')
self.url = reverse("api_databases:all")
def test_no_access_for_anonymous_user(self):
response = self.client.post(self.url)
self.checkResponse(response, 403)
def test_no_access_for_non_admin_user(self):
self.client.login(username='jackdoe', password='1234')
self.client.login(username="jackdoe", password=TEST_PWD)
response = self.client.post(self.url)
self.checkResponse(response, 403)
def test_create_database_failure(self):
self.client.login(username=settings.SYSTEM_ACCOUNT, password='1234')
response = self.client.post(self.url,
json.dumps({
'name': self.db_name,
'declaration': self.DATABASE
}), content_type='application/json')
data = self.checkResponse(response, 400)
self.client.login(username=settings.SYSTEM_ACCOUNT, password=TEST_PWD)
response = self.client.post(
self.url,
json.dumps({"name": self.db_name, "declaration": self.DATABASE}),
content_type="application/json",
)
self.checkResponse(response, 400)
def test_create_database(self):
# Create a dataformat
(dataformat, errors) = DataFormat.objects.create_dataformat(self.system_user, 'float', '')
assert dataformat, errors
(dataformat, errors) = DataFormat.objects.create_dataformat(
self.system_user, "float", ""
)
self.assertIsNotNone(dataformat, errors)
dataformat.share()
self.client.login(username=settings.SYSTEM_ACCOUNT, password='1234')
self.client.login(username=settings.SYSTEM_ACCOUNT, password=TEST_PWD)
response = self.client.post(self.url,
json.dumps({
'name': self.db_name,
'declaration': self.DATABASE
}), content_type='application/json')
response = self.client.post(
self.url,
json.dumps({"name": self.db_name, "declaration": self.DATABASE}),
content_type="application/json",
)
data = self.checkResponse(response, 201, content_type='application/json')
data = self.checkResponse(response, 201, content_type="application/json")
self.assertTrue(data['name'] == self.db_name)
self.assertTrue(data["name"] == self.db_name)
databases = Database.objects.all()
self.assertEqual(databases.count(), 1)
......@@ -127,24 +128,29 @@ class DatabaseCreationAPI(DatabaseAPIBase):
class DatabaseRetrievalAPI(DatabaseAPIBase):
def test_retrieve_database(self):
(dataformat, errors) = DataFormat.objects.create_dataformat(self.system_user, 'float', '')
assert dataformat, errors
(dataformat, errors) = DataFormat.objects.create_dataformat(
self.system_user, "float", ""
)
self.assertIsNotNone(dataformat, errors)
dataformat.share()
(database, errors) = Database.objects.create_database(self.db_name, declaration=self.DATABASE)
assert database, errors
(database, errors) = Database.objects.create_database(
self.db_name, declaration=self.DATABASE
)
self.assertIsNotNone(database, errors)
database.share()
self.client.login(username=settings.SYSTEM_ACCOUNT, password='1234')
self.client.login(username=settings.SYSTEM_ACCOUNT, password=TEST_PWD)
url = reverse('api_databases:object', kwargs={'database_name': self.db_name, 'version': 1})
url = reverse(
"api_databases:object", kwargs={"database_name": self.db_name, "version": 1}
)
response = self.client.get(url, format='json')
data = self.checkResponse(response, 200, content_type='application/json')
response = self.client.get(url, format="json")
data = self.checkResponse(response, 200, content_type="application/json")
declaration = json.loads(data['declaration'])
self.assertTrue(declaration['root_folder'].startswith('/path_to_db_folder'))
declaration = json.loads(data["declaration"])
self.assertTrue(declaration["root_folder"].startswith("/path_to_db_folder"))
database.delete()
\ No newline at end of file
database.delete()
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -53,6 +53,7 @@ LOGGING['loggers']['beat.core']['handlers'] = ['discard']
LOGGING['loggers']['beat.web']['handlers'] = ['discard']
LOGGING['loggers']['beat.web.utils.management.commands']['handlers'] = ['discard']
BASE_DIR = os.path.dirname(os.path.abspath(__name__))
PREFIX = os.environ.get('BEAT_TEST_PREFIX', os.path.realpath('./test_prefix'))
ALGORITHMS_ROOT = os.path.join(PREFIX, 'algorithms')
PLOTTERS_ROOT = os.path.join(PREFIX, 'plotters')
......
This diff is collapsed.
This diff is collapsed.
......@@ -3,7 +3,7 @@ extends = common.cfg
develop = .
[sources]
beat.examples = git git@gitlab.idiap.ch:beat/beat.examples egg=false
beat.examples = git https://gitlab.idiap.ch/beat/beat.examples egg=false
[node]
npms += protractor
......
[buildout]
parts = scripts node bower help django
parts = scripts node bower django
extensions = mr.developer
auto-checkout = *
newest = false
......@@ -51,10 +51,3 @@ packages = jquery#~1.11.3
angular-ui-codemirror
executable = ${buildout:bin-directory}/bower --config.interactive=false
downloads = static
[help]