Commit 235798e9 authored by Philip ABBET's avatar Philip ABBET

Refactoring of the 'DataFormat' class (which was duplicated in beat.core)

parent d58add06
......@@ -36,11 +36,43 @@ import six
import numpy
import simplejson
from . import utils
from .baseformat import baseformat
class Storage(utils.Storage):
"""Resolves paths for dataformats
prefix (str): Establishes the prefix of your installation.
name (str): The name of the dataformat object in the format
def __init__(self, prefix, name):
if name.count('/') != 2:
raise RuntimeError("invalid dataformat name: `%s'" % name)
self.username,, self.version = name.split('/')
self.fullname = name
path = utils.hashed_or_simple(prefix, 'dataformats', name)
super(Storage, self).__init__(path)
def hash(self):
"""The 64-character hash of the database declaration JSON"""
return super(Storage, self).hash('#description')
class DataFormat(object):
"""Data formats define the chunks of data that circulate at data formats.
"""Data formats define the chunks of data that circulate between blocks.
......@@ -67,6 +99,17 @@ class DataFormat(object):
name (str): The full, valid name of this dataformat
description (str): The short description string, loaded from the JSON
file if one was set.
documentation (str): The full-length docstring for this object.
storage (object): A simple object that provides information about file
paths for this dataformat
errors (list of str): A list containing errors found while loading this
data (dict): The original data for this dataformat, as loaded by our JSON
......@@ -83,23 +126,44 @@ class DataFormat(object):
def __init__(self, prefix, data, parent=None, dataformat_cache=None):
self._name = None = None
self.resolved = None
self.prefix = prefix
self.errors = [] = None
self.resolved = None
self.referenced = {}
self.parent = parent
# if the user has not provided a cache, still use one for performance
dataformat_cache = dataformat_cache if dataformat_cache is not None else {}
self._load(data, dataformat_cache)
if self._name is not None: #registers it into the cache, even if failed
dataformat_cache[self._name] = self
def _load(self, data, dataformat_cache):
"""Loads the dataformat"""
if isinstance(data, dict): = 'analysis:result'
self._name = 'analysis:result' = data
else: = data
json_path = os.path.join(prefix, 'dataformats', data + '.json')
with open(json_path, 'rb') as f: = simplejson.load(f)
self._name = data = Storage(self.prefix, data)
json_path =
if not
self.errors.append('Dataformat declaration file not found: %s' % json_path)
dataformat_cache[] = self #registers itself into the cache
with open(json_path, 'rb') as f: = simplejson.load(f)
dataformat_cache[self._name] = self #registers itself into the cache
self.resolved = copy.deepcopy(
......@@ -152,7 +216,7 @@ class DataFormat(object):
if '#extends' in self.resolved:
ext =['#extends']
self.referenced[ext] = maybe_load_format(, ext, dataformat_cache)
self.referenced[ext] = maybe_load_format(self._name, ext, dataformat_cache)
basetype = self.resolved['#extends']
tmp = self.resolved
self.resolved = basetype.resolved
......@@ -160,6 +224,23 @@ class DataFormat(object):
del self.resolved['#extends'] #avoids infinite recursion
def name(self):
"""Returns the name of this object, either from the filename or composed
from the hierarchy it belongs.
if self.parent and self._name is None:
return self.parent[0].name + '.' + self.parent[1] + '_type'
return self._name or '__unnamed_dataformat__'
def name(self, value):
self._name = value = Storage(self.prefix, value)
def schema_version(self):
"""Returns the schema version"""
......@@ -245,6 +326,57 @@ class DataFormat(object):
def valid(self):
return not bool(self.errors)
def description(self):
"""The short description for this object"""
return'#description', None)
def description(self, value):
"""Sets the short description for this object"""['#description'] = value
def documentation(self):
"""The full-length description for this object"""
if not self._name:
raise RuntimeError("dataformat has no name")
return None
def documentation(self, value):
"""Sets the full-length description for this object"""
if not self._name:
raise RuntimeError("dataformat has no name")
if hasattr(value, 'read'):
def hash(self):
"""Returns the hexadecimal hash for its declaration"""
if not self._name:
raise RuntimeError("dataformat has no name")
def validate(self, data):
"""Validates a piece of data provided by the user
......@@ -292,7 +424,9 @@ class DataFormat(object):
if other.extends:
if == other.extends: return True
else: return self.isparent(other.referenced[other.extends])
if == other.extends:
return True
return self.isparent(other.referenced[other.extends])
return False
......@@ -34,6 +34,7 @@ import simplejson
import collections
import copy
import six
import os
def _sha256(s):
......@@ -63,6 +64,12 @@ def _stringify(dictionary):
def toUserPath(username):
hash = _sha256(username)
return os.path.join(hash[0:2], hash[2:4], username)
def hash(dictionary_or_string):
if isinstance(dictionary_or_string, dict):
return _sha256(_stringify(dictionary_or_string))
......@@ -37,6 +37,21 @@ import six
from . import hash
def hashed_or_simple(prefix, what, path, suffix='.json'):
"""Returns a hashed path or simple path depending on where the resource is"""
username, right_bit = path.split('/', 1)
hashed_prefix = hash.toUserPath(username)
candidate = os.path.join(prefix, what, hashed_prefix, right_bit)
if os.path.exists(candidate + suffix):
return candidate
return os.path.join(prefix, what, path)
def safe_rmfile(f):
"""Safely removes a file from the disk"""
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment