Commit 6e24f0ce authored by André Anjos's avatar André Anjos 💬
Browse files

Merge branch 'update-ci' into 'master'

Update CI and documentation linking

See merge request !272
parents 945a6c3b 7a90346b
Pipeline #27613 failed with stages
in 19 minutes and 32 seconds
# This build file uses template features from YAML so it is generic enough for
# any Bob project. Don't modify it unless you know what you're doing.
# Definition of global variables (all stages)
variables:
CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
DOCKER_REGISTRY: docker.idiap.ch
TEST_PREFIX_PATH_FILE: 'test_prefix_$CI_JOB_ID.txt'
BOOTSTRAP: "https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/bootstrap.py"
# Definition of our build pipeline order
# This build does not deploy to PyPI
stages:
- build
- deploy
# Build targets
.build_template: &build_job
build_linux_36:
stage: build
before_script:
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci master #installs ci support scripts
- ./_ci/before_build.sh
- docker info
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN docker.idiap.ch
variables:
PYTHON_VERSION: "3.6"
TEST_PREFIX_PATH_FILE: 'test_prefix_$CI_JOB_ID.txt'
tags:
- docker-build
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- export BEAT_TEST_PREFIX=`mktemp -d --tmpdir=/var/tmp beat_test_prefix.XXXXXXXXX`
- echo $BEAT_TEST_PREFIX > $TEST_PREFIX_PATH_FILE
- ./scripts/before_build.sh
script:
- ./scripts/build.sh
after_script:
- ./scripts/after_build.sh
- ./_ci/after_build.sh
- conda env create -f dev.yml
- conda activate beat-web
- buildout
- python -c "from beat.core.test.utils import pull_docker_test_images as f; f()"
- export COVERAGE_FILE=.coverage.django
- ./bin/coverage run --source=${CI_PROJECT_NAME} ./bin/django test --settings=beat.web.settings.ci -v 2
- export BEAT_CMDLINE_TEST_PLATFORM="django://beat.web.settings.ci"
- export COVERAGE_FILE=.coverage.cmdline
- export NOSE_WITH_COVERAGE=1
- export NOSE_COVER_PACKAGE=beat.web
- ./bin/coverage run --source=beat.cmdline ./bin/nosetests -sv beat.cmdline
- unset COVERAGE_FILE
- ./bin/coverage combine .coverage.django .coverage.cmdline
- ./bin/coverage report
- ./bin/sphinx-apidoc --separate -d 2 --output=doc/api ${CI_PROJECT_NAMESPACE} beat/web/*/migrations beat/web/*/tests
- ./bin/sphinx-build doc/api sphinx/api
- ./bin/sphinx-build doc/admin sphinx/admin
- ./bin/sphinx-build doc/user sphinx/user
- conda deactivate
- export BEAT_TEST_PREFIX=`cat $TEST_PREFIX_PATH_FILE`
- rm -rf ${BEAT_TEST_PREFIX} ${TEST_PREFIX_PATH_FILE}
- conda activate base
- bdt ci clean -vv
artifacts:
expire_in: 1 week
paths:
- sphinx
cache: &build_caches
key: "linux-cache"
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
.build_linux_template: &linux_build_job
<<: *build_job
tags:
- docker-build
cache:
<<: *build_caches
key: "linux-cache"
build_linux_36:
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.6"
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/*.zip
- sphinx
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/deploy.sh
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- bdt ci deploy -vv
- bdt ci clean -vv
dependencies:
- build_linux_36
cache:
<<: *build_caches
tags:
- deployer
- docker
deploy_beta:
......
This diff is collapsed.
......@@ -28,7 +28,7 @@
from django.contrib.auth.models import User
from ...libraries.models import Library
from ...common.testutils import tearDownModule
from ...common.testutils import tearDownModule # noqa test runner will call it
from ..models import Algorithm
......@@ -38,18 +38,25 @@ from .core import AlgorithmsBaseTestCase
class DependencyTests(AlgorithmsBaseTestCase):
def setUp(self):
super(DependencyTests, self).setUp()
user = User.objects.create_user('johndoe', 'johndoe@test.org', '1234')
user = User.objects.get(username="johndoe")
algorithm_declaration = """{
"description": "some description",
"schema_version": 1,
"language": "python",
"splittable": false,
"groups": [
{
"name": "channel1",
"inputs": {
"in_data": {
"type": "johndoe/integer/1"
}
},
"outputs": {
"out_data": {
"type": "johndoe/integer/1"
}
}
}
],
......@@ -81,44 +88,42 @@ class DependencyTests(AlgorithmsBaseTestCase):
"""
(algorithm, errors) = Algorithm.objects.create_algorithm(
author=user,
name='algorithm1',
name="algorithm1",
declaration=algorithm_declaration,
code=algorithm_code,
)
assert algorithm, errors
self.assertIsNotNone(algorithm, errors)
(library, errors) = Library.objects.create_library(
author=user,
name='library1',
name="library1",
declaration=library_declaration,
code=library_code,
)
assert library, errors
self.assertIsNotNone(library, errors)
def test_add_dependency(self):
algorithm1 = Algorithm.objects.get(name="algorithm1")
library1 = Library.objects.get(name='library1')
library1 = Library.objects.get(name="library1")
decl = algorithm1.declaration
decl.setdefault('uses', {})
decl['uses']['dep1'] = library1.fullname()
decl.setdefault("uses", {})
decl["uses"]["dep1"] = library1.fullname()
algorithm1.declaration = decl
algorithm1.save()
self.assertEqual(algorithm1.referenced_libraries.count(), 1)
def test_update_dependency(self):
algorithm1 = Algorithm.objects.get(name="algorithm1")
library1 = Library.objects.get(name='library1')
library1 = Library.objects.get(name="library1")
decl = algorithm1.declaration
decl.setdefault('uses', {})
decl['uses']['dep1'] = library1.fullname()
decl['uses']['dep2'] = library1.fullname()
decl.setdefault("uses", {})
decl["uses"]["dep1"] = library1.fullname()
decl["uses"]["dep2"] = library1.fullname()
algorithm1.declaration = decl
algorithm1.save()
......@@ -130,7 +135,7 @@ class DependencyTests(AlgorithmsBaseTestCase):
self.assertEqual(algorithm1.referenced_libraries.count(), 1)
decl = algorithm1.declaration
decl['uses'] = {}
decl["uses"] = {}
algorithm1.declaration = decl
algorithm1.save()
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -25,20 +25,21 @@
# #
###############################################################################
import os
import beat.core.hash
from django.test import TestCase
from django.conf import settings
from ...utils.management.commands import install
from ...experiments.models import Experiment
from ...experiments.models import Block
from ...experiments.models import CachedFile
from ...algorithms.models import Algorithm
from ...common.testutils import tearDownModule # noqa test runner will call it
from ..models import Queue
from ..models import Worker
from ..models import Environment
from ..models import Job
from ..models import JobSplit
from ..utils import setup_backend
from ..management.commands import qsetup
......@@ -46,106 +47,80 @@ from ..management.commands import qsetup
from beat.core.dataformat import DataFormat
from beat.core.data import CachedDataSink
from beat.core.database import Database
import beat.core.hash
import os
import sys
#----------------------------------------------------------
# ----------------------------------------------------------
ONE_QUEUE_TWO_WORKERS = {
"queues": {
"queue": {
"memory-limit": 4*1024,
"time-limit": 1440, #1 day
"memory-limit": 4 * 1024,
"time-limit": 1440, # 1 day
"cores-per-slot": 1,
"max-slots-per-user": 2,
"environments": [
'Python 2.7 (1.3.0)'
],
"environments": ["Python 2.7 (1.3.0)"],
"slots": {
'node1': {
"quantity": 1,
"priority": 0
},
'node2': {
"quantity": 1,
"priority": 0
}
"node1": {"quantity": 1, "priority": 0},
"node2": {"quantity": 1, "priority": 0},
},
"groups": [
"Default",
],
"groups": ["Default"],
}
},
"workers": {
"node1": {
"cores": 1,
"memory": 16*1024,
},
"node2": {
"cores": 1,
"memory": 16*1024,
}
"node1": {"cores": 1, "memory": 16 * 1024},
"node2": {"cores": 1, "memory": 16 * 1024},
},
"environments": {
"Python 2.7 (1.3.0)": {
"name": 'Python 2.7',
"version": '1.3.0',
"name": "Python 2.7",
"version": "1.3.0",
"short_description": "Test",
"description": "Test environment",
"languages": ["python"],
},
}
},
}
#----------------------------------------------------------
# ----------------------------------------------------------
class BackendUtilitiesMixin(object):
@classmethod
def setup_test_data(cls):
install.create_sites()
system_user, plot_user, user = install.create_users('user', 'user')
install.add_group('Default')
system_user, plot_user, user = install.create_users("user", "user")
install.add_group("Default")
setup_backend(qsetup.DEFAULT_CONFIGURATION)
Worker.objects.update(active=True)
env = Environment.objects.get(name='Python 2.7')
env = Environment.objects.get(name="Python 2.7")
queue = Queue.objects.first()
template_data = dict(
system_user = system_user,
plot_user = plot_user,
user = user,
private = False,
queue = queue.name,
environment = dict(name=env.name, version=env.version),
system_user=system_user,
plot_user=plot_user,
user=user,
private=False,
queue=queue.name,
environment=dict(name=env.name, version=env.version),
)
prefix = os.path.join(
os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0]))),
'src',
'beat.examples',
)
source_prefix = os.path.join(settings.BASE_DIR, "src", "beat.examples")
install.install_contributions(prefix, 'system', template_data)
install.install_contributions(prefix, 'test', template_data)
install.install_contributions(source_prefix, "system", template_data)
install.install_contributions(source_prefix, "test", template_data)
if not os.path.exists(settings.CACHE_ROOT):
os.mkdir(settings.CACHE_ROOT)
os.mkdir(settings.CACHE_ROOT)
def clean_cache(self):
for p, dirs, files in os.walk(settings.CACHE_ROOT, topdown=False):
files = [f for f in files if not f.startswith('.')]
dirs[:] = [d for d in dirs if not d.startswith('.')] #note: in-place
files = [f for f in files if not f.startswith(".")]
dirs[:] = [d for d in dirs if not d.startswith(".")] # note: in-place
for f in files:
fullpath = os.path.join(p, f)
......@@ -155,9 +130,9 @@ class BackendUtilitiesMixin(object):
fullpath = os.path.join(p, d)
os.rmdir(fullpath)
def set_experiment_state(self, experiment, experiment_status=None, block_status=None,
cache_status=None):
def set_experiment_state(
self, experiment, experiment_status=None, block_status=None, cache_status=None
):
if block_status:
for name, status in block_status.items():
block = experiment.blocks.get(name=name)
......@@ -175,9 +150,8 @@ class BackendUtilitiesMixin(object):
experiment.status = experiment_status
experiment.save()
def generate_cached_files(self, hash, splits):
dataformat = DataFormat(settings.PREFIX, 'system/integer/1')
dataformat = DataFormat(settings.PREFIX, "system/integer/1")
path = os.path.join(settings.CACHE_ROOT, beat.core.hash.toPath(hash))
os.makedirs(os.path.dirname(path))
......@@ -200,80 +174,76 @@ class BackendUtilitiesMixin(object):
start = indices[0]
end = indices[1]
sink.write({
'value': value,
},
start_data_index = start,
end_data_index = end
)
sink.write({"value": value}, start_data_index=start, end_data_index=end)
value += 1
sink.close()
def prepare_databases(self, configuration):
for _, cfg in configuration['datasets'].items():
path = beat.core.hash.toPath(beat.core.hash.hashDataset(
cfg['database'], cfg['protocol'], cfg['set']), suffix='.db')
for _, cfg in configuration["datasets"].items():
path = beat.core.hash.toPath(
beat.core.hash.hashDataset(
cfg["database"], cfg["protocol"], cfg["set"]
),
suffix=".db",
)
if not os.path.exists(os.path.join(settings.CACHE_ROOT, path)):
database = Database(settings.PREFIX, cfg['database'])
view = database.view(cfg['protocol'], cfg['set'])
database = Database(settings.PREFIX, cfg["database"])
view = database.view(cfg["protocol"], cfg["set"])
view.index(os.path.join(settings.CACHE_ROOT, path))
#----------------------------------------------------------
# ----------------------------------------------------------
class BaseBackendTestCase(TestCase, BackendUtilitiesMixin):
@classmethod
def setUpTestData(cls):
cls.setup_test_data()
cls.setup_test_data()
def setUp(self):
self.clean_cache()
def tearDown(self):
self.clean_cache()
def check_single(self, xp):
'''Checks user/user/single/1/single'''
"""Checks user/user/single/1/single"""
self.assertEqual(xp.blocks.count(), 2)
b0 = xp.blocks.all()[0]
self.assertEqual(b0.name, 'echo')
self.assertEqual(b0.name, "echo")
self.assertEqual(b0.status, Block.PENDING)
self.assertEqual(b0.algorithm, Algorithm.objects.get(name='integers_echo'))
self.assertEqual(b0.algorithm, Algorithm.objects.get(name="integers_echo"))
self.assertEqual(b0.dependencies.count(), 0)
self.assertEqual(b0.dependents.count(), 1)
self.assertEqual(b0.queue.name, 'queue')
self.assertEqual(b0.environment.name, 'Python 2.7')
self.assertEqual(b0.queue.name, "queue")
self.assertEqual(b0.environment.name, "Python 2.7")
self.assertEqual(b0.required_slots, 1)
self.assertEqual(b0.inputs.count(), 1)
self.assertEqual(b0.outputs.count(), 1)
self.assertEqual(b0.job.splits.count(), 0) #not scheduled yet
self.assertEqual(b0.job.splits.count(), 0) # not scheduled yet
assert not b0.done()
self.assertFalse(b0.done())
b1 = xp.blocks.all()[1]
self.assertEqual(b1.name, 'analysis')
self.assertEqual(b1.name, "analysis")
self.assertEqual(b1.status, Block.PENDING)
self.assertEqual(b1.algorithm, Algorithm.objects.get(name='integers_echo_analyzer'))
self.assertEqual(
b1.algorithm, Algorithm.objects.get(name="integers_echo_analyzer")
)
self.assertEqual(b1.dependencies.count(), 1)
self.assertEqual(b1.dependents.count(), 0)
self.assertEqual(b1.queue.name, 'queue')
self.assertEqual(b1.environment.name, 'Python 2.7')
self.assertEqual(b1.queue.name, "queue")
self.assertEqual(b1.environment.name, "Python 2.7")
self.assertEqual(b1.required_slots, 1)
self.assertEqual(b1.inputs.count(), 1)
self.assertEqual(b1.outputs.count(), 1)
self.assertEqual(b1.job.splits.count(), 0) #not scheduled yet
self.assertEqual(b1.job.splits.count(), 0) # not scheduled yet
assert not b1.done()
self.assertFalse(b1.done())
......@@ -25,25 +25,24 @@
# #
###############################################################################
from ...common.testutils import BaseTestCase as APITestCase
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from ...common.testutils import BaseTestCase as APITestCase
from ...common.testutils import tearDownModule # noqa test runner will call it
class CancelAllExperimentsAPI(APITestCase):
class CancelAllExperimentsAPI(APITestCase):
def setUp(self):
self.url = reverse('backend:cancel-experiments')
self.url = reverse("backend:cancel-experiments")
def test_no_access_for_anonymous_user(self):
response = self.client.get(self.url)
self.checkResponse(response, 302) #redirects to login page
self.checkResponse(response, 302) # redirects to login page
def test_no_access_for_non_superuser(self):
User.objects.create_user('johndoe', 'johndoe@test.org', '1234')
self.client.login(username='johndoe', password='1234')
PASSWORD = "1234"
User.objects.create_user("johndoe", "johndoe@test.org", PASSWORD)
self.client.login(username="johndoe", password=PASSWORD)
response = self.client.get(self.url)
self.checkResponse(response, 403)
......@@ -26,7 +26,6 @@
###############################################################################
import os
import sys
import time
import shutil
import tempfile
......@@ -34,43 +33,36 @@ import tempfile
from django.core import management
from django.test import TestCase
from ...common.testutils import tearDownModule # noqa test runner will call it
from ..utils import cleanup_cache
class CacheCleanUp(TestCase):
def setUp(self):
self.cache = tempfile.mkdtemp(prefix='beat_')
self.cache = tempfile.mkdtemp(prefix="beat_")
def tearDown(self):
shutil.rmtree(self.cache)
def touch(self, f, times=None):
"""Replicates the `touch' command-line utility"""
with open(f, 'a'): os.utime(f, times)
with open(f, "a"):
os.utime(f, times)
def J(self, *args):
return os.path.join(*((self.cache,) + args))
def prepare_cleanup_full(self):