install.py 38.2 KB
Newer Older
André Anjos's avatar
André Anjos committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

###############################################################################
#                                                                             #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
# Contact: beat.support@idiap.ch                                              #
#                                                                             #
# This file is part of the beat.web module of the BEAT platform.              #
#                                                                             #
# Commercial License Usage                                                    #
# Licensees holding valid commercial BEAT licenses may use this file in       #
# accordance with the terms contained in a written agreement between you      #
# and Idiap. For further information contact tto@idiap.ch                     #
#                                                                             #
# Alternatively, this file may be used under the terms of the GNU Affero      #
# Public License version 3 as published by the Free Software and appearing    #
# in the file LICENSE.AGPL included in the packaging of this file.            #
# The BEAT platform is distributed in the hope that it will be useful, but    #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
#                                                                             #
# You should have received a copy of the GNU Affero Public License along      #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
#                                                                             #
###############################################################################


"""
Examples:

  To install advanced databases, formats, libraries, algorithms, toolchains,
  experiments and plotters:

35
    $ ./bin/django install -v1 advanced
André Anjos's avatar
André Anjos committed
36 37 38 39

  To install only test databases, formats, libraries, algorithms, toolchains,
  experiments and plotters:

40
    $ ./bin/django install -v1 test
André Anjos's avatar
André Anjos committed
41 42 43 44 45 46 47 48 49 50 51

  Note: If you need to specify your own path to the directories containing the
  databases, you could just create a simple JSON file as follows::

    {
      "atnt/1": "/remote/databases/atnt",
      "banca/2": "/remote/databases/banca"
    }

  Then just use the previous script with the option ``--database-root-file``::

52
      $ ./bin/django install -v1 --database-root-file=<file.json>
André Anjos's avatar
André Anjos committed
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100

  By default, paths to the root of all databases are set to match the Idiap
  Research Institute filesystem organisation.
"""

import logging
logger = logging.getLogger(__name__)

import pkg_resources

import os
import sys
import fnmatch
import collections
import simplejson

from django.core.management.base import BaseCommand
from django.conf import settings

import beat.core.dataformat
import beat.core.database
import beat.core.library
import beat.core.algorithm
import beat.core.plotter
import beat.core.toolchain
import beat.core.experiment


def add_user(name, passwd, token_key):
    """Adds a simple platform user"""

    from django.contrib.auth.models import User
    from rest_framework.authtoken.models import Token

    try:
        user = User.objects.get(username=name)
    except User.DoesNotExist:
        user = User()
        user.username = name
        user.first_name = name.capitalize()
        user.last_name = name.capitalize() + 'son'
        user.email = '%s@example.com' % name
        user.is_active = True
        if passwd is not None:
            user.set_password(passwd)
            user.is_staff = True
            user.is_superuser = True
        user.save()
101 102 103 104 105 106
        #set profile
        user.profile.status = 'A'
        user.profile.rejection_date = None
        user.profile.supervision_key = None
        user.profile.save()
        user.save()
André Anjos's avatar
André Anjos committed
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138

        if name == passwd:
            logger.info("Created user `%s' with password `%s'", name, passwd)
        elif passwd is None:
            logger.info("Created user `%s' without password", name)
        else:
            logger.info("Created user `%s' with (non-default) password", name)
    else:
        logger.info("Creation of user `%s' skipped: already exists", name)

    user.auth_token.delete()
    token = Token(user=user)
    token.key = token_key
    token.save()

    return user


def add_group(name):

    from django.contrib.auth.models import Group

    group, created = Group.objects.get_or_create(name=name)

    if created:
        logger.info("Created group `%s'", name)
    else:
        logger.info("Creation of group `%s' skipped: already exists", name)

    return group


139
def setup_environment(queue_config_filename, verbosity):
André Anjos's avatar
André Anjos committed
140

141 142
    from django.core.management import call_command
    call_command('qsetup', verbosity=verbosity, reset=True,
Philip ABBET's avatar
Philip ABBET committed
143
                 config=queue_config_filename)
André Anjos's avatar
André Anjos committed
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161


def create_sites():

    from django.contrib.sites.models import Site

    def _setup_site(pk, name, domain):
        obj = Site.objects.get_or_create(pk=pk)[0]
        obj.name = name
        obj.domain = domain
        obj.save()
        logger.info('Saved site %s (pk=%s)' % (obj, obj.pk))

    _setup_site(1, 'Development Server', '127.0.0.1:8000')
    _setup_site(2, 'Staging System', 'beatweb-staging')
    _setup_site(3, 'Production System', 'www.beat-eu.org')


162 163 164 165 166 167 168 169 170 171
def create_users(username, passwd):

    # Sets up initial users, if not already there.
    system_user = add_user(settings.SYSTEM_ACCOUNT, None, '1')
    plot_user = add_user(settings.PLOT_ACCOUNT, None, '2')
    user = add_user(username, passwd, '3')

    return system_user, plot_user, user


André Anjos's avatar
André Anjos committed
172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
def list_objects(prefix, project, category, fnfilter):
    """Lists all objects matching a certain filter"""

    path = os.path.join(prefix, project, category)

    if not os.path.exists(path): return []

    retval = []
    for base, dirs, files in os.walk(path):
        for k in fnmatch.filter(files, fnfilter):
            retval.append(os.path.join(base, k).replace(path + os.sep, ''))
            retval[-1] = os.path.splitext(retval[-1])[0]

    return retval


def detemplatize(template, data):
    """Loads the json file and replaces the templates


    Parameters:

      template (str): The template to de-templatize using Jinja2

      data (dict): A dictionary containing the values that will be available to
        the template resolution.


    Returns:

      str: Contains the resolved template

    """

    from jinja2 import Environment, DictLoader
    env = Environment(loader=DictLoader({'object': template}))
    return env.get_template('object').render(**data)


def upload_dataformat(prefix, name, data):
    """Uploads the dataformat template to the platform, under a certain user


    Parameters:

      prefix (str): The prefix where the dataformat template is sitting

      name (str): The canonical name of the dataformat

      data (dict): A dictionary with template substitutions


    Returns:

      bool: Indicates if the operation was succesful

    """

    storage = beat.core.dataformat.Storage(prefix, name)
    if not storage.exists(): return False #should be ignored
    declaration = detemplatize(storage.json.load(), data)
    obj = simplejson.loads(declaration)
    description = storage.doc.load() if storage.doc.exists() else ''

    # Uploads the data format into the platform
    from ....dataformats.models import DataFormat

    author = data[name.split(os.sep)[0].replace('name', '')]

    dataformat = DataFormat.objects.filter(
Philip ABBET's avatar
Philip ABBET committed
242 243 244 245
        author=author,
        name=storage.name,
        version=int(storage.version),
    )
André Anjos's avatar
André Anjos committed
246 247 248 249 250 251 252 253 254 255

    if not dataformat:
        # creates it
        (dataformat, errors) = DataFormat.objects.create_dataformat(
            author=author,
            name=storage.name,
            version=int(storage.version),
            short_description=obj.get('description', ''),
            description=description,
            declaration=declaration,
Philip ABBET's avatar
Philip ABBET committed
256
        )
André Anjos's avatar
André Anjos committed
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292
        if dataformat is None:
            raise SyntaxError(errors)

        logger.info("Added dataformat `%s'", dataformat)

    else:
        dataformat = dataformat[0]
        dataformat.short_description = obj.get('description', '')
        dataformat.description = description
        dataformat.save()
        logger.info("Updated dataformat `%s'", dataformat)

    if not data['private']:
        dataformat.share()
        logger.info("Set dataformat `%s' as public", dataformat)

    return True


def load_database_folders(filename):
    """Simply loads the json file describing the database root folders

    :param str filename:
      The path to the filename to be loaded.

    :return:
      A dictionary, containing the JSON data. If the file does not exist, then
      the returned dictionary is empty.
    """

    if os.path.exists(filename):
        return simplejson.loads(open(filename,'rb').read())

    return {}


293
def upload_database(prefix, name, data, new_only=False):
André Anjos's avatar
André Anjos committed
294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
    """Uploads the database template to the platform


    Parameters:

      prefix (str): The prefix where the object template is sitting

      name (str): The canonical name of the object

      data (dict): A dictionary with template substitutions


    Returns:

      bool: Indicates if the operation was succesful

    """

    storage = beat.core.database.Storage(prefix, name)
    if not storage.exists():
        return False #should be ignored
    declaration = detemplatize(storage.json.load(), data)
    obj = simplejson.loads(declaration, object_pairs_hook=collections.OrderedDict)
    description = storage.doc.load() if storage.doc.exists() else ''
    code = storage.code.load() if storage.code and storage.code.exists() else ''

    if 'root_folder' in data:
        obj['root_folder'] = data['root_folder']
        declaration = simplejson.dumps(obj, indent=4)

    from ....databases.models import Database
    from ....common.models import Shareable

    database = Database.objects.filter(name=storage.name,
Philip ABBET's avatar
Philip ABBET committed
328
                                       version=int(storage.version))
André Anjos's avatar
André Anjos committed
329 330 331 332 333 334 335 336 337 338

    if not database:

        (database, errors) = Database.objects.create_database(
            name = storage.name,
            declaration = declaration,
            code = code,
            short_description = obj.get('description', ''),
            description = description,
            version=int(storage.version),
Philip ABBET's avatar
Philip ABBET committed
339
        )
André Anjos's avatar
André Anjos committed
340 341 342 343 344 345 346

        if database is None:
            logger.warn("Did not add database `%s', because: %s", name, errors)
            return False
        else:
            logger.info("Added database `%s'", database)

347 348
    elif new_only:
        return True
André Anjos's avatar
André Anjos committed
349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400
    else: #only updates files

        database = database[0]
        database.short_description = obj.get('description', '')
        database.declaration = declaration
        database.description = description
        database.code = code
        database.save()

        logger.info("Updated database `%s'", database)

    if not data['private']:
        database.sharing = Shareable.PUBLIC
        database.save()
        logger.info("Set database `%s' as public", database)

    return True


def upload_toolchain(prefix, name, data):
    """Uploads the toolchain to the running platform


    Parameters:

      prefix (str): The prefix where the object template is sitting

      name (str): The canonical name of the object

      data (dict): A dictionary with template substitutions


    Returns:

      bool: Indicates if the operation was succesful

    """

    storage = beat.core.toolchain.Storage(prefix, name)
    if not storage.exists(): return False #should be ignored
    declaration = detemplatize(storage.json.load(), data)
    obj = simplejson.loads(declaration)
    description = storage.doc.load() if storage.doc.exists() else ''

    from ....toolchains.models import Toolchain

    author = data[name.split(os.sep)[0].replace('name', '')]

    toolchain = Toolchain.objects.filter(
        author=author,
        name=storage.name,
        version=int(storage.version),
Philip ABBET's avatar
Philip ABBET committed
401
    )
André Anjos's avatar
André Anjos committed
402 403 404 405 406 407 408 409 410 411

    if not toolchain:

        (toolchain, errors) = Toolchain.objects.create_toolchain(
            author=author,
            name=storage.name,
            version=int(storage.version),
            short_description=obj.get('description', ''),
            description=description,
            declaration=declaration,
Philip ABBET's avatar
Philip ABBET committed
412
        )
André Anjos's avatar
André Anjos committed
413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
        if toolchain is None:
            logger.warn("Did not add toolchain `%s', because: %s", name, errors)
            return False
        elif errors:
            logger.warn("Added faulty toolchain `%s': %s", toolchain, errors)
        else:
            logger.info("Added toolchain `%s'", toolchain)

    else: #only updates files

        toolchain = toolchain[0]
        toolchain.short_description = obj.get('description', '')
        toolchain.declaration = declaration
        toolchain.description = description
        toolchain.save()
        logger.info("Updated toolchain `%s'", toolchain)

    if not data['private']:
        toolchain.share()
        logger.info("Set toolchain `%s' as public", toolchain)

    return True


def upload_library(prefix, name, data):
    """Uploads the library to a running platform


    Parameters:

      prefix (str): The prefix where the object template is sitting

      name (str): The canonical name of the object

      data (dict): A dictionary with template substitutions


    Returns:

      bool: Indicates if the operation was successful

    """

    storage = beat.core.library.Storage(prefix, name)
    if not storage.json.exists(): return False #should be ignored
    declaration = detemplatize(storage.json.load(), data)
    obj = simplejson.loads(declaration)
    storage.language = obj.get('language', 'python')
    description = storage.doc.load() if storage.doc.exists() else ''
    code = storage.code.load() if storage.code and storage.code.exists() else ''

    from ....libraries.models import Library

    author = data[name.split(os.sep)[0].replace('name', '')]

    library = Library.objects.filter(
        author=author,
        name=storage.name,
        version=int(storage.version),
Philip ABBET's avatar
Philip ABBET committed
472
    )
André Anjos's avatar
André Anjos committed
473 474 475 476 477 478 479 480 481 482

    if not library:
        (library, errors) = Library.objects.create_library(
            author=author,
            name=storage.name,
            version=int(storage.version),
            short_description=obj.get('description', ''),
            description=description,
            declaration=declaration,
            code=code,
Philip ABBET's avatar
Philip ABBET committed
483
        )
André Anjos's avatar
André Anjos committed
484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541
        if library is None:
            logger.warn("Did not add library `%s', because: %s", name, errors)
            return False
        else:
            logger.info("Added library `%s'", library)

    else: #only updates files

        library = library[0]
        library.short_description = obj.get('description', '')
        library.declaration = declaration
        library.description = description
        library.code = code
        library.save()
        logger.info("Updated library `%s'", library)

    if not data['private']:
        library.share(public=True)
        logger.info("Set library `%s' as public", library)

    return True


def upload_algorithm(prefix, name, data):
    """Uploads the algorithm to a running platform


    Parameters:

      prefix (str): The prefix where the object template is sitting

      name (str): The canonical name of the object

      data (dict): A dictionary with template substitutions


    Returns:

      bool: Indicates if the operation was succesful

    """

    storage = beat.core.algorithm.Storage(prefix, name)
    if not storage.json.exists(): return False #should be ignored
    declaration = detemplatize(storage.json.load(), data)
    obj = simplejson.loads(declaration)
    storage.language = obj.get('language', 'python')
    description = storage.doc.load() if storage.doc.exists() else ''
    code = storage.code.load() if storage.code and storage.code.exists() else ''

    from ....algorithms.models import Algorithm

    author = data[name.split(os.sep)[0].replace('name', '')]

    algorithm = Algorithm.objects.filter(
        author=author,
        name=storage.name,
        version=int(storage.version),
Philip ABBET's avatar
Philip ABBET committed
542
    )
André Anjos's avatar
André Anjos committed
543 544 545 546 547 548 549 550 551 552

    if not algorithm:
        (algorithm, errors) = Algorithm.objects.create_algorithm(
            author=author,
            name=storage.name,
            version=int(storage.version),
            short_description=obj.get('description', ''),
            description=description,
            declaration=declaration,
            code=code,
Philip ABBET's avatar
Philip ABBET committed
553
        )
André Anjos's avatar
André Anjos committed
554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607
        if algorithm is None:
            logger.warn("Did not add algorithm `%s', because: %s", name, errors)
            return False
        else:
            logger.info("Added algorithm `%s'", algorithm)

    else: #only updates files

        algorithm = algorithm[0]
        algorithm.short_description = obj.get('description', '')
        algorithm.declaration = declaration
        algorithm.description = description
        algorithm.save()
        logger.info("Updated algorithm `%s'", algorithm)

    if not data['private']:
        algorithm.share(public=True)
        logger.info("Set algorithm `%s' as public", algorithm)

    return True


def upload_experiment(prefix, name, data):
    """Uploads the experiment to the running platform


    Parameters:

      prefix (str): The prefix where the object template is sitting

      name (str): The canonical name of the object

      data (dict): A dictionary with template substitutions


    Returns:

      bool: Indicates if the operation was succesful

    """

    storage = beat.core.experiment.Storage(prefix, name)
    if not storage.exists(): return False #should be ignored
    declaration = detemplatize(storage.json.load(), data)
    obj = simplejson.loads(declaration)
    storage.language = obj.get('language', 'python')
    description = storage.doc.load() if storage.doc.exists() else ''

    from ....toolchains.models import Toolchain

    author = data[name.split(os.sep)[0].replace('name', '')]

    toolchain_storage = beat.core.toolchain.Storage(name, storage.toolchain)
    toolchain = Toolchain.objects.get(
Philip ABBET's avatar
Philip ABBET committed
608 609 610 611
        author=author,
        name=toolchain_storage.name,
        version=int(toolchain_storage.version),
    )
André Anjos's avatar
André Anjos committed
612 613 614 615 616 617 618

    from ....experiments.models import Experiment

    experiment = Experiment.objects.filter(
        author=author,
        toolchain=toolchain,
        name=storage.name,
Philip ABBET's avatar
Philip ABBET committed
619
    )
André Anjos's avatar
André Anjos committed
620 621 622 623 624 625 626 627 628

    if not experiment:
        (experiment, _toolchain_obj, errors) = Experiment.objects.create_experiment(
            author=author,
            toolchain=toolchain,
            name=storage.name,
            declaration=declaration,
            short_description=obj.get('description', ''),
            description=description,
Philip ABBET's avatar
Philip ABBET committed
629
        )
André Anjos's avatar
André Anjos committed
630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678
        if experiment is None:
            logger.warn("Did not add experiment `%s', because: %s", name, errors)
            return False
        else:
            logger.info("Added experiment `%s'", experiment)

    else: #only updates files

        experiment = experiment[0]
        experiment.short_description = obj.get('description', '')
        experiment.declaration = declaration
        experiment.description = description
        experiment.save()
        logger.info("Updated experiment `%s'", experiment)

    if not data['private']:
        experiment.share()
        logger.info("Set experiment `%s' as public", experiment)

    return True


def upload_plotter(prefix, name, data):
    """Uploads the plotter to a running platform


    Parameters:

      prefix (str): The prefix where the object template is sitting

      name (str): The canonical name of the object

      data (dict): A dictionary with template substitutions


    Returns:

      bool: Indicates if the operation was succesful

    """

    storage = beat.core.plotter.Storage(prefix, name)
    if not storage.json.exists(): return False #should be ignored
    declaration = detemplatize(storage.json.load(), data)
    obj = simplejson.loads(declaration)
    storage.language = obj.get('language', 'python')
    description = storage.doc.load() if storage.doc.exists() else ''
    code = storage.code.load() if storage.code and storage.code.exists() else ''

679 680
    from ....plotters.models import Plotter, DefaultPlotter, PlotterParameter
    from ....common.models import Shareable
André Anjos's avatar
André Anjos committed
681 682 683 684 685 686 687

    author = data[name.split(os.sep)[0].replace('name', '')]

    plotter = Plotter.objects.filter(
        author=author,
        name=storage.name,
        version=int(storage.version),
Philip ABBET's avatar
Philip ABBET committed
688
    )
André Anjos's avatar
André Anjos committed
689

690
    sample_data_file_location = prefix + "/plotters/" + name.split("/")[0] + "/" + \
Philip ABBET's avatar
Philip ABBET committed
691
        name.split("/")[1] + "/sample_data.txt"
692 693 694 695

    with open(sample_data_file_location) as sample_data_file:
        sample_data = simplejson.load(sample_data_file)

André Anjos's avatar
André Anjos committed
696 697 698 699 700 701 702 703 704
    if not plotter:
        (plotter, errors) = Plotter.objects.create_plotter(
            author=author,
            name=storage.name,
            version=int(storage.version),
            short_description=obj.get('description', ''),
            description=description,
            declaration=declaration,
            code=code,
Philip ABBET's avatar
Philip ABBET committed
705
        )
André Anjos's avatar
André Anjos committed
706 707 708 709
        if plotter is None:
            logger.warn("Did not add plotter `%s', because: %s", name, errors)
            return False
        else:
710 711
            plotter.sample_data = simplejson.dumps(sample_data, indent=4)
            plotter.save()
André Anjos's avatar
André Anjos committed
712 713 714 715 716 717 718 719 720
            logger.info("Added plotter `%s'", plotter)

    else: #only updates documentation

        plotter = plotter[0]
        plotter.short_description = obj.get('description', '')
        plotter.declaration = declaration
        plotter.description = description
        plotter.code = code
721
        plotter.sample_data = simplejson.dumps(sample_data, indent=4)
André Anjos's avatar
André Anjos committed
722 723 724 725 726 727 728 729 730 731 732 733
        plotter.save()
        logger.info("Updated plotter `%s'", plotter)

    if not data['private']:
        plotter.share(public=True)
        logger.info("Set plotter `%s' as public", plotter)

    # Make it the format default
    if plotter.dataformat.author.username == author.username and \
            plotter.dataformat.name == storage.name and \
            plotter.dataformat.version == int(storage.version):

734 735 736 737 738 739 740 741 742 743 744 745 746
        # Adding some plotter parameters
        plotterparameter_data_file_location = prefix + "/plotters/" + name.split("/")[0] + "/" + \
            name.split("/")[1] + "/default_plotterparameter.txt"

        short_desc_file_location = prefix + "/plotters/" + name.split("/")[0] + "/" + \
            name.split("/")[1] + "/default_plotterparameter_short_description.txt"

        with open(plotterparameter_data_file_location) as plotterparameter_data_file:
            plotterparameter_data = simplejson.load(plotterparameter_data_file)

        with open(short_desc_file_location) as short_desc_data_file:
            short_desc = short_desc_data_file.readline().split("\n")[0]

747
        plotterparameter = PlotterParameter.objects.create(name=plotter.dataformat.name,
Philip ABBET's avatar
Philip ABBET committed
748 749
                                                           author=author, plotter=plotter, data=simplejson.dumps(plotterparameter_data,
                                                                                                                 indent=4), short_description=short_desc, sharing = Shareable.PUBLIC)
750

751 752 753 754
        plotterparameter.save()
        logger.info("Add plotterparameter `%s' ", plotterparameter)


André Anjos's avatar
André Anjos committed
755 756 757 758 759
        default = DefaultPlotter.objects.filter(dataformat=plotter.dataformat)

        if default:
            default.plotter = plotter
        else:
760
            default = DefaultPlotter(dataformat=plotter.dataformat,
Philip ABBET's avatar
Philip ABBET committed
761
                                     plotter=plotter, parameter=plotterparameter)
André Anjos's avatar
André Anjos committed
762 763
            default.save()

764 765 766 767 768
        logger.info("Set plotter `%s' and plotterparameter `%s'  as default for `%s'", plotter, plotterparameter, plotter.dataformat)

        if plotter.dataformat.name == "isoroc":
            # Adding extra plotterparameter if not already present for plotter isoroc
            other_plotterparameter_location = prefix + "/plotters/" + name.split("/")[0] + "/" + \
Philip ABBET's avatar
Philip ABBET committed
769
                "other_plotterparameters"
770 771

            the_folders = filter(lambda x:\
Philip ABBET's avatar
Philip ABBET committed
772 773
                                 os.path.isdir(os.path.join(other_plotterparameter_location, x)),\
                                 os.listdir(other_plotterparameter_location))
774 775 776 777 778 779 780

            for folder_name in the_folders:

                others_plotterparameter = PlotterParameter.objects.filter(
                    author=author,
                    name=folder_name,
                    version=int(storage.version),
Philip ABBET's avatar
Philip ABBET committed
781
                )
782 783 784 785 786 787 788 789 790 791 792 793 794

                if others_plotterparameter is not None:
                    param_folder = other_plotterparameter_location + "/" + folder_name
                    data_file_location = param_folder + "/default_plotterparameter.txt"
                    short_desc_file_location = param_folder + "/default_plotterparameter_short_description.txt"

                    with open(data_file_location) as plotterparameter_data_file:
                        plotterparameter_data = simplejson.load(plotterparameter_data_file)

                    with open(short_desc_file_location) as short_desc_data_file:
                        short_desc = short_desc_data_file.readline().split("\n")[0]

                    plotterparameter = PlotterParameter.objects.create(name=folder_name,
Philip ABBET's avatar
Philip ABBET committed
795 796
                                                                       author=author, plotter=plotter, data=simplejson.dumps(plotterparameter_data,
                                                                                                                             indent=4), short_description=short_desc, sharing = Shareable.PUBLIC)
André Anjos's avatar
André Anjos committed
797

798
                    logger.info("Add plotterparameter `%s' ", folder_name)
André Anjos's avatar
André Anjos committed
799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830

    return True


def upload_dispatcher(prefix, project, type, name, data):
    """Uploads the experiment to the running platform


    Parameters:

      prefix (str): The path leading to the project prefixes

      project (str): The sub-directory where the object template is sitting

      type (str): The type of object to upload (one of ``dataformats``,
        ``databases``, ``libraries``, ``algorithms``, ``toolchains``,
        ``experiments`` or ``plot_templates``)

      name (str): The canonical name of the object

      data (dict): A dictionary with template substitutions


    Returns:

      bool: Indicates if the operation was succesful

    """

    base_subdir = os.path.join(prefix, project)

    valid_types = {
Philip ABBET's avatar
Philip ABBET committed
831
        'dataformats': upload_dataformat,
André Anjos's avatar
André Anjos committed
832 833 834 835 836 837
            'databases': upload_database,
            'libraries': upload_library,
            'algorithms': upload_algorithm,
            'toolchains': upload_toolchain,
            'experiments': upload_experiment,
            'plotters': upload_plotter,
Philip ABBET's avatar
Philip ABBET committed
838
    }
André Anjos's avatar
André Anjos committed
839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868

    if type not in valid_types:
        raise KeyError("Type must be one of `%s'" % ', '.join(valid_types.keys()))

    upload_function = valid_types[type]

    try:

        status = upload_function(base_subdir, name, data)
        if not status:
            logger.warn("Skipping `%s/%s/%s': disabled", project, type, name)
            return False

    except Exception as e:
        logger.warn("Not adding `%s/%s/%s': %s", project, type, name, e)
        return False

    return True


def link_database_versions():
    '''Link object versions together'''

    from ....databases.models import Database

    for obj in Database.objects.all():
        if obj.version > 1:
            #search for similar
            try:
                existing = Database.objects.get(name=obj.name,
Philip ABBET's avatar
Philip ABBET committed
869
                                                version=obj.version-1)
870
                logger.info("Linking database `%s' -> `%s' (version)",
Philip ABBET's avatar
Philip ABBET committed
871
                            obj, existing)
André Anjos's avatar
André Anjos committed
872 873 874 875 876 877 878 879 880 881 882 883 884 885 886
                obj.previous_version = existing
                obj.save()
            except Database.DoesNotExist:
                pass #ignores


def link_contribution_versions(klass):
    '''Link object versions together'''


    for obj in klass.objects.all():
        if obj.version > 1:
            #search for similar
            try:
                existing = klass.objects.get(author=obj.author, name=obj.name,
Philip ABBET's avatar
Philip ABBET committed
887
                                             version=obj.version-1)
888
                logger.info("Linking %s `%s' -> `%s' (version)",
Philip ABBET's avatar
Philip ABBET committed
889
                            klass.__name__.lower(), obj, existing)
André Anjos's avatar
André Anjos committed
890 891 892 893 894 895
                obj.previous_version = existing
                obj.save()
            except klass.DoesNotExist:
                pass #ignores


896
def install_contributions(source_prefix, project, template_data,
Philip ABBET's avatar
Philip ABBET committed
897
                          db_root_file=None):
898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922
    '''Installs all contributions for a given project


    Parameters:

      source_prefix (str): The path to the base directory containing the
        projects where objects must be installed from.

      project (str): The project within the ``source_prefix`` where to install
        objects from.

      template_data (dict): A dictionary containing standard template data for
        completing template objects installed on the project.

      db_root_file (str, Optional): Optional path to a JSON describing the
        database root for databases to be inserted. Database names not present
        at the project directory will be ignored.

    '''

    # Dataformat adding requires a special trick as there are dependencies
    # between different dataformats. Our recipe: we try to upload all of them
    # one after the other. If one fails, we retry on the next loop, until all
    # formats have been uploaded.
    dataformat_filenames_next = list_objects(source_prefix, project,
Philip ABBET's avatar
Philip ABBET committed
923
                                             'dataformats', '*.json')
924 925 926 927 928 929 930 931 932 933
    dataformat_filenames_cur = []

    while True:
        if not dataformat_filenames_next: break
        if len(dataformat_filenames_cur) == len(dataformat_filenames_next):
            break
        dataformat_filenames_cur = dataformat_filenames_next
        dataformat_filenames_next = []
        for k in dataformat_filenames_cur:
            if not upload_dispatcher(source_prefix, project, 'dataformats', k,
Philip ABBET's avatar
Philip ABBET committed
934
                                     template_data):
935 936 937 938 939 940 941 942 943 944 945
                dataformat_filenames_next.append(k)
    from ....dataformats.models import DataFormat
    link_contribution_versions(DataFormat)

    # Reads database root file, if provided
    db_root = {}
    if db_root_file: db_root.update(load_database_folders(db_root_file))

    for k in list_objects(source_prefix, project, 'databases', '*.json'):
        if k in db_root: template_data['root_folder'] = db_root[k]
        upload_dispatcher(source_prefix, project, 'databases', k,
Philip ABBET's avatar
Philip ABBET committed
946
                          template_data)
947 948 949
    link_database_versions()

    for k in list_objects(source_prefix, project, 'toolchains',
Philip ABBET's avatar
Philip ABBET committed
950
                          '*.json'):
951
        upload_dispatcher(source_prefix, project, 'toolchains', k,
Philip ABBET's avatar
Philip ABBET committed
952
                          template_data)
953 954 955 956 957 958 959
    from ....toolchains.models import Toolchain
    link_contribution_versions(Toolchain)

    # Libraries adding requires a special trick as there are
    # dependencies between different libraries and algorithms. Our
    # recipe: we use the same technique as for dataformats.
    library_filenames_next = list_objects(source_prefix, project,
Philip ABBET's avatar
Philip ABBET committed
960
                                          'libraries', '*.json')
961 962 963 964 965 966 967 968 969 970
    library_filenames_cur = []

    while True:
        if not library_filenames_next: break
        if len(library_filenames_cur) == len(library_filenames_next):
            break
        library_filenames_cur = library_filenames_next
        library_filenames_next = []
        for k in library_filenames_cur:
            if not upload_dispatcher(source_prefix, project,
Philip ABBET's avatar
Philip ABBET committed
971
                                     'libraries', k, template_data):
972 973 974 975 976
                library_filenames_next.append(k)
    from ....libraries.models import Library
    link_contribution_versions(Library)

    for k in list_objects(source_prefix, project, 'algorithms',
Philip ABBET's avatar
Philip ABBET committed
977
                          '*.json'):
978
        upload_dispatcher(source_prefix, project, 'algorithms', k,
Philip ABBET's avatar
Philip ABBET committed
979
                          template_data)
980 981 982 983 984
    from ....algorithms.models import Algorithm
    link_contribution_versions(Algorithm)

    for k in list_objects(source_prefix, project, 'plotters', '*.json'):
        upload_dispatcher(source_prefix, project, 'plotters', k,
Philip ABBET's avatar
Philip ABBET committed
985
                          template_data)
986 987 988 989
    from ....plotters.models import Plotter
    link_contribution_versions(Plotter)

    for k in list_objects(source_prefix, project, 'experiments',
Philip ABBET's avatar
Philip ABBET committed
990
                          '*.json'):
991
        upload_dispatcher(source_prefix, project, 'experiments', k,
Philip ABBET's avatar
Philip ABBET committed
992
                          template_data)
993

André Anjos's avatar
André Anjos committed
994 995 996 997 998 999 1000 1001 1002 1003 1004

class Command(BaseCommand):

    help = 'Uploads stock contributions into the database'


    def __init__(self):

        super(Command, self).__init__()

        self.prefix = os.path.join(
Philip ABBET's avatar
Philip ABBET committed
1005 1006
            os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0]))),
            'src',
André Anjos's avatar
André Anjos committed
1007
              'beat.examples',
Philip ABBET's avatar
Philip ABBET committed
1008
        )
André Anjos's avatar
André Anjos committed
1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025

        # gets a list of all available projects, excluding directories
        ignore = ['system', 'LICENSE', '.git', '.gitignore', 'README.rst']
        projects = os.listdir(self.prefix)
        self.projects = []
        for candidate in projects:
            if (candidate not in ignore) or (candidate not in projects):
                self.projects.append(candidate)


    def add_arguments(self, parser):

        from argparse import RawDescriptionHelpFormatter
        parser.epilog = __doc__
        parser.formatter_class = RawDescriptionHelpFormatter

        parser.add_argument('--username', '-u', dest='username', type=str,
Philip ABBET's avatar
Philip ABBET committed
1026 1027
                            default='user', help='Username to create, associated with ' \
                            'user contributions [default: %(default)s]')
André Anjos's avatar
André Anjos committed
1028 1029

        parser.add_argument('--password', '-P', dest='password', type=str,
Philip ABBET's avatar
Philip ABBET committed
1030 1031
                            default='user', help='The password to set for such an user ' \
                            '[default: %(default)s]')
André Anjos's avatar
André Anjos committed
1032 1033

        parser.add_argument('--private', '-p', action='store_true',
Philip ABBET's avatar
Philip ABBET committed
1034 1035
                            dest='private', default=False, help='Set this flag if all ' \
                            'objects should be private to the user rather than public')
André Anjos's avatar
André Anjos committed
1036 1037

        parser.add_argument('--database-root-file', '-R', type=str,
Philip ABBET's avatar
Philip ABBET committed
1038 1039 1040 1041 1042
                            dest='database_root_file', help='The JSON file containing ' \
                            'the root directories of the databases installed ' \
                            'on the platform. If not set or if there is no ' \
                            'entry in this file for a given database, the root ' \
                            'directory defined on the JSON database file is used.')
André Anjos's avatar
André Anjos committed
1043 1044

        parser.add_argument('--source-prefix', '-X', type=str,
Philip ABBET's avatar
Philip ABBET committed
1045 1046 1047 1048
                            dest='source_prefix', default=self.prefix,
                            help='Set this to the root of the directory containing ' \
                            'the project prefixes you wish to install ' \
                            '[default: %(default)s]')
André Anjos's avatar
André Anjos committed
1049 1050

        parser.add_argument('--queue-configuration', '-Q', type=str,
Philip ABBET's avatar
Philip ABBET committed
1051 1052 1053 1054 1055
                            dest='queue_configuration', help='The configuration for ' \
                            'queues and environments to be inserted into the ' \
                            'web server. If not passed, use the default ' \
                            'queue-worker-environment configuration for a ' \
                            'local development server.')
André Anjos's avatar
André Anjos committed
1056 1057

        parser.add_argument('project', nargs='*', type=str,
Philip ABBET's avatar
Philip ABBET committed
1058 1059 1060 1061
                            default=self.projects, help='The project data you wish to ' \
                            'install. Currently, the default is to install ' \
                            'data for all available projects. [default: ' \
                            '%s]' % ', '.join(self.projects))
André Anjos's avatar
André Anjos committed
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078


    def handle(self, *ignored, **arguments):

        # Setup this command's logging level
        global logger
        arguments['verbosity'] = int(arguments['verbosity'])
        if arguments['verbosity'] >= 1:
            if arguments['verbosity'] == 1: logger.setLevel(logging.INFO)
            elif arguments['verbosity'] >= 2: logger.setLevel(logging.DEBUG)

        # Checks projects
        if not arguments['project']:
            arguments['project'] = ['test', 'advanced']

        while 'system' in arguments['project']:
            logger.warn("Removing `system' from the list of projects to " \
Philip ABBET's avatar
Philip ABBET committed
1079
                        "install (this is them minimal default anyway)")
André Anjos's avatar
André Anjos committed
1080 1081 1082 1083 1084
            arguments['project'].remove('system')

        for k in arguments['project']:
            if k not in self.projects:
                logger.error("Project `%s' is not available, choose from: %s",
Philip ABBET's avatar
Philip ABBET committed
1085
                             ', '.join("`%s'" % k for k in self.projects))
André Anjos's avatar
André Anjos committed
1086 1087 1088 1089 1090 1091 1092 1093 1094 1095
                sys.exit(1)

        # Creates the prefix directory
        if not os.path.exists(settings.PREFIX):
            logger.info("Creating prefix directory `%s'...", settings.PREFIX)
            os.makedirs(settings.PREFIX)

        # Creates the cache directory
        if not os.path.exists(settings.CACHE_ROOT):
            logger.info("Creating cache directory `%s'...",
Philip ABBET's avatar
Philip ABBET committed
1096
                        settings.CACHE_ROOT)
André Anjos's avatar
André Anjos committed
1097 1098 1099 1100
            os.makedirs(settings.CACHE_ROOT)

        # Sync database
        from django.core.management import call_command
1101
        call_command('migrate', interactive=False, verbosity=1)
André Anjos's avatar
André Anjos committed
1102 1103 1104 1105

        # Setup sites: 1.Development; 2.Staging; 3.Production
        create_sites()

1106
        system_user, plot_user, user = create_users(arguments['username'],
Philip ABBET's avatar
Philip ABBET committed
1107
                                                    arguments['password'])
1108

André Anjos's avatar
André Anjos committed
1109 1110 1111 1112 1113

        # Sets up initial groups
        add_group('Default')

        # Sets up the queue and environments
1114
        setup_environment(arguments['queue_configuration'],
Philip ABBET's avatar
Philip ABBET committed
1115
                          arguments['verbosity'])
1116 1117 1118 1119
        from ....backend.models import Environment, EnvironmentLanguage, Queue
        from ....code.models import Code
        environment = EnvironmentLanguage.objects.filter(language=Code.PYTHON).first().environment
        queue = environment.queues.first()
André Anjos's avatar
André Anjos committed
1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130

        # Iterates over projects to install
        for project in ['system'] + arguments['project']:

            template_data = dict(
                system_user = system_user,
                plot_user = plot_user,
                user = user,
                private = arguments['private'],
                queue = queue.name,
                environment = dict(name=environment.name,
Philip ABBET's avatar
Philip ABBET committed
1131 1132
                                   version=environment.version),
            )
André Anjos's avatar
André Anjos committed
1133 1134 1135

            logger.info("Adding objects for project `%s'...", project)

1136
            install_contributions(self.prefix, project, template_data,
Philip ABBET's avatar
Philip ABBET committed
1137
                                  arguments['database_root_file'])