Skip to content
Snippets Groups Projects
Commit b52ceda9 authored by André Anjos's avatar André Anjos :speech_balloon:
Browse files

Merge branch 'issue_11' into 'master'

Issue 11

See merge request !10
parents d1ec7b41 bdf4b8a2
No related branches found
No related tags found
1 merge request!10Issue 11
Pipeline #
...@@ -61,22 +61,47 @@ def dbshell_command(subparsers): ...@@ -61,22 +61,47 @@ def dbshell_command(subparsers):
def upload(arguments): def upload(arguments):
"""For SQLite databases: uploads the db.sql3 database file to a server.""" """Uploads generated metadata to the Idiap build server"""
# get the file name of the target db
assert len(arguments.files) == 1 import pkg_resources
assert os.path.basename(arguments.files[0]) == 'db.sql3' basedir = pkg_resources.resource_filename('bob.db.%s' % arguments.name, '')
source_file = arguments.files[0] assert basedir, "Database and package names do not match. Your declared " \
target_file = os.path.join(arguments.destination, arguments.name + ".tar.bz2") "database name should be <name>, if your package is called bob.db.<name>"
if os.path.exists(source_file): target_file = os.path.join(arguments.destination,
print ("Compressing file '%s' to '%s'" %(source_file, target_file)) arguments.name + ".tar.bz2")
import tarfile, stat
f = tarfile.open(target_file, 'w:bz2') # check all files exist
f.add(source_file, os.path.basename(source_file)) for p in arguments.files:
f.close() if not os.path.exists(p):
os.chmod(target_file, stat.S_IRUSR|stat.S_IWUSR | stat.S_IRGRP|stat.S_IWGRP | stat.S_IROTH) raise IOError("Metadata file `%s' is not available. Did you run " \
else: "`create' before attempting to upload?" % (p,))
print ("WARNING! Database file '%s' is not available. Did you run 'bob_dbmanage %s create' ?" % (source_file, arguments.name))
# if destination exists, try to erase it before
if os.path.exists(target_file):
try:
os.unlink(target_file)
except Exception as e:
print("Cannot erase existing file `%s': %s" % (target_file, e))
# if you get here, all files are there, ready to package
print("Compressing metadata files to `%s'" % (target_file,))
# compress
import tarfile
f = tarfile.open(target_file, 'w:bz2')
for k,p in enumerate(arguments.files):
n = os.path.relpath(p, basedir)
print("+ [%d/%d] %s" % (k+1, len(arguments.files), n))
f.add(p, n)
f.close()
# set permissions for sane Idiap storage
import stat
perms = stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IWGRP|stat.S_IROTH
os.chmod(target_file, perms)
def upload_command(subparsers): def upload_command(subparsers):
"""Adds a new 'upload' subcommand to your parser""" """Adds a new 'upload' subcommand to your parser"""
...@@ -89,41 +114,81 @@ def upload_command(subparsers): ...@@ -89,41 +114,81 @@ def upload_command(subparsers):
def download(arguments): def download(arguments):
"""For SQLite databases: Downloads the db.sql3 database file from a server.""" """Downloads and uncompresses meta data generated files from Idiap
# get the file name of the target db
assert len(arguments.files) == 1
assert os.path.basename(arguments.files[0]) == 'db.sql3'
target_file = arguments.files[0]
if os.path.exists(target_file) and not arguments.force:
print ("Skipping download of file '%s' since it exists already." % target_file)
else:
# get URL of database file
source_url = os.path.join(arguments.source, arguments.name + ".tar.bz2")
# download
import sys, tempfile, tarfile
if sys.version_info[0] <= 2:
import urllib2 as urllib
else:
import urllib.request as urllib
Parameters:
arguments (argparse.Namespace): A set of arguments passed by the
command-line parser
Returns:
int: A POSIX compliant return value of ``0`` if the download is successful,
or ``1`` in case it is not.
"""
# check all files don't exist
for p in arguments.files:
if os.path.exists(p):
if arguments.force:
os.unlink(p)
else:
raise IOError("Metadata file `%s' is already available. Please " \
"remove self-generated files before attempting download or " \
"--force" % (p,))
# if you get here, all files aren't there, unpack
source_url = os.path.join(arguments.source, arguments.name + ".tar.bz2")
target_dir = arguments.test_dir #test case
if not target_dir: #tries to dig it up
import pkg_resources
try: try:
print ("Extracting url '%s' to '%s'" %(source_url, target_file)) target_dir = pkg_resources.resource_filename('bob.db.%s' % \
u = urllib.urlopen(source_url) arguments.name, '')
f = tempfile.NamedTemporaryFile(suffix = ".tar.bz2") except ImportError as e:
open(f.name, 'wb').write(u.read()) print("The package `bob.db.%s' is not currently installed" % \
t = tarfile.open(fileobj=f, mode = 'r:bz2') (arguments.name,))
t.extract(os.path.basename(target_file), os.path.dirname(target_file)) print("N.B.: The database and package names **must** match. Your " \
t.close() "package should be named `bob.db.%s', if the driver name for your "
f.close() "database is `<name>'")
return False return 1
except Exception as e:
print ("Error while downloading: '%s'" % e) # download file from Idiap server, unpack and remove it
return True import sys
import tempfile
import tarfile
import pkg_resources
if sys.version_info[0] <= 2:
import urllib2 as urllib
else:
import urllib.request as urllib
try:
print ("Extracting url `%s' into `%s'" %(source_url, target_dir))
u = urllib.urlopen(source_url)
f = tempfile.NamedTemporaryFile(suffix = ".tar.bz2")
open(f.name, 'wb').write(u.read())
t = tarfile.open(fileobj=f, mode='r:bz2')
t.extractall(target_dir)
t.close()
f.close()
return 0
except Exception as e:
print ("Error while downloading: %s" % e)
return 1
def download_command(subparsers): def download_command(subparsers):
"""Adds a new 'download' subcommand to your parser""" """Adds a new 'download' subcommand to your parser"""
from argparse import SUPPRESS
if 'DOCSERVER' in os.environ: if 'DOCSERVER' in os.environ:
USE_SERVER=os.environ['DOCSERVER'] USE_SERVER=os.environ['DOCSERVER']
else: else:
...@@ -133,6 +198,7 @@ def download_command(subparsers): ...@@ -133,6 +198,7 @@ def download_command(subparsers):
parser.add_argument("--source", parser.add_argument("--source",
default="%s/software/bob/databases/latest/" % USE_SERVER) default="%s/software/bob/databases/latest/" % USE_SERVER)
parser.add_argument("--force", action='store_true', help = "Overwrite existing database files?") parser.add_argument("--force", action='store_true', help = "Overwrite existing database files?")
parser.add_argument("--test-dir", help=SUPPRESS)
parser.set_defaults(func=download) parser.set_defaults(func=download)
return parser return parser
...@@ -174,58 +240,86 @@ def version_command(subparsers): ...@@ -174,58 +240,86 @@ def version_command(subparsers):
@six.add_metaclass(abc.ABCMeta) @six.add_metaclass(abc.ABCMeta)
class Interface(object): class Interface(object):
"""Base manager for Bob databases""" """Base manager for Bob databases
You should derive and implement an Interface object on every ``bob.db``
package you create.
"""
@abc.abstractmethod @abc.abstractmethod
def name(self): def name(self):
'''Returns a simple name for this database, w/o funny characters, spaces''' '''The name of this database
Returns:
str: a Python-conforming name for this database. This **must** match the
package name. If the package is named ``bob.db.foo``, then this function
must return ``foo``.
'''
return return
@abc.abstractmethod @abc.abstractmethod
def files(self): def files(self):
'''Returns a python iterable with all auxiliary files needed. '''Files containing meta-data for this package
Returns:
list: A python iterable with all metadata files needed. The paths listed
by this method should correspond to full paths (not relative ones) w.r.t.
the database package implementing it. This is normally achieved by using
``pkg_resources.resource_filename()``.
The values should be take w.r.t. where the python file that declares the
database is sitting at.
''' '''
return return
@abc.abstractmethod @abc.abstractmethod
def version(self): def version(self):
'''Returns the current version number defined in setup.py''' '''The version of this package
Returns:
str: The current version number defined in ``setup.py``
'''
return return
@abc.abstractmethod @abc.abstractmethod
def type(self): def type(self):
'''Returns the type of auxiliary files you have for this database '''The type of auxiliary files you have for this database
If you return 'sqlite', then we append special actions such as 'dbshell' Returns:
on 'bob_dbmanage.py' automatically for you. Otherwise, we don't.
If you use auxiliary text files, just return 'text'. We may provide str: A string defining the type of database implemented. You can return
special services for those types in the future. only two values on this function, either ``sqlite`` or ``text``. If you
return ``sqlite``, then we append special actions such as ``dbshell`` on
``bob_dbmanage`` automatically for you. Otherwise, we don't.
Use the special name 'builtin' if this database is an integral part of Bob.
''' '''
return return
def setup_parser(self, parser, short_description, long_description): def setup_parser(self, parser, short_description, long_description):
'''Sets up the base parser for this database. '''Sets up the base parser for this database.
Keyword arguments: Parameters:
short_description (str): A short description (one-liner) for this
database
long_description (str): A more involved explanation of this database
short_description
A short description (one-liner) for this database
long_description Returns:
A more involved explanation of this database
argparse.ArgumentParser: a subparser, ready so you can add commands on
Returns a subparser, ready to be added commands on
''' '''
from argparse import RawDescriptionHelpFormatter from argparse import RawDescriptionHelpFormatter
...@@ -248,11 +342,13 @@ class Interface(object): ...@@ -248,11 +342,13 @@ class Interface(object):
# adds some stock commands # adds some stock commands
version_command(subparsers) version_command(subparsers)
if type in ('sqlite',): if files:
dbshell_command(subparsers)
upload_command(subparsers) upload_command(subparsers)
download_command(subparsers) download_command(subparsers)
if type in ('sqlite',):
dbshell_command(subparsers)
if files is not None: if files is not None:
files_command(subparsers) files_command(subparsers)
...@@ -261,7 +357,7 @@ class Interface(object): ...@@ -261,7 +357,7 @@ class Interface(object):
@abc.abstractmethod @abc.abstractmethod
def add_commands(self, parser): def add_commands(self, parser):
'''Adds commands to a given (:py:mod:`argparse`) parser. '''Adds commands to a given :py:class:`argparse.ArgumentParser`
This method, effectively, allows you to define special commands that your This method, effectively, allows you to define special commands that your
database will be able to perform when called from the common driver like database will be able to perform when called from the common driver like
...@@ -269,16 +365,20 @@ class Interface(object): ...@@ -269,16 +365,20 @@ class Interface(object):
You are not obliged to overwrite this method. If you do, you will have the You are not obliged to overwrite this method. If you do, you will have the
chance to establish your own commands. You don't have to worry about stock chance to establish your own commands. You don't have to worry about stock
commands such as :py:meth:`files` or :py:meth:`version`. They will be automatically commands such as :py:meth:`files` or :py:meth:`version`. They will be
hooked-in depending on the values you return for :py:meth:`type` and automatically hooked-in depending on the values you return for
:py:meth:`files`. :py:meth:`type` and :py:meth:`files`.
Keyword arguments Parameters:
parser (argparse.ArgumentParser): An instance of a parser that you can
customize, i.e., call :py:meth:`argparse.ArgumentParser.add_argument`
on.
parser
An instance of a :py:class:`argparse.ArgumentParser` that you can customize, i.e., call
:py:meth:`argparse.ArgumentParser.add_argument` on.
''' '''
return return
__all__ = ('Interface',) __all__ = ('Interface',)
...@@ -18,12 +18,12 @@ def test_download_01(): ...@@ -18,12 +18,12 @@ def test_download_01():
tmpdir = tempfile.mkdtemp() tmpdir = tempfile.mkdtemp()
try: try:
arguments = Namespace(files=[tmpdir+'/db.sql3'], force=False, arguments = Namespace(files=[tmpdir+'/db.sql3'], force=False,
name='non_existing_db', type='sqlite', version='0.9.0', name='does_not_exist', test_dir=tmpdir, version='0.9.0',
source='%s/software/bob/databases/latest/' % USE_SERVER) source='%s/software/bob/databases/latest/' % USE_SERVER)
assert download(arguments) == True assert download(arguments) == 1 #error #error #error #error
arguments = Namespace(files=[tmpdir+'/db.sql3'], force=False, arguments = Namespace(files=[tmpdir+'/db.sql3'], force=False,
name='biowave_test', type='sqlite', version='0.9.0', name='banca', test_dir=tmpdir, version='0.9.0',
source='%s/software/bob/databases/latest/' % USE_SERVER) source='%s/software/bob/databases/latest/' % USE_SERVER)
assert download(arguments) == False assert download(arguments) == 0 #success
finally: finally:
shutil.rmtree(tmpdir) shutil.rmtree(tmpdir)
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
import os import os
class null(object): class null(object):
"""A look-alike stream that discards the input""" """A look-alike stream that discards the input"""
...@@ -21,6 +22,7 @@ class null(object): ...@@ -21,6 +22,7 @@ class null(object):
pass pass
def apsw_is_available(): def apsw_is_available():
"""Checks lock-ability for SQLite on the current file system""" """Checks lock-ability for SQLite on the current file system"""
...@@ -39,8 +41,19 @@ def apsw_is_available(): ...@@ -39,8 +41,19 @@ def apsw_is_available():
# if you get to this point, all seems OK # if you get to this point, all seems OK
return True return True
class SQLiteConnector(object): class SQLiteConnector(object):
'''An object that handles the connection to SQLite databases.''' '''An object that handles the connection to SQLite databases.
Parameters:
filename (str): The name of the file containing the SQLite database
readonly (bool): Should I try and open the database in read-only mode?
lock (str): Any vfs name as output by apsw.vfsnames()
'''
@staticmethod @staticmethod
def filesystem_is_lockable(database): def filesystem_is_lockable(database):
...@@ -62,20 +75,8 @@ class SQLiteConnector(object): ...@@ -62,20 +75,8 @@ class SQLiteConnector(object):
APSW_IS_AVAILABLE = apsw_is_available() APSW_IS_AVAILABLE = apsw_is_available()
def __init__(self, filename, readonly=False, lock=None):
"""Initializes the connector
Keyword arguments def __init__(self, filename, readonly=False, lock=None):
filename
The name of the file containing the SQLite database
readonly
Should I try and open the database in read-only mode?
lock
Any vfs name as output by apsw.vfsnames()
"""
self.readonly = readonly self.readonly = readonly
self.vfs = lock self.vfs = lock
...@@ -87,6 +88,7 @@ class SQLiteConnector(object): ...@@ -87,6 +88,7 @@ class SQLiteConnector(object):
import warnings import warnings
warnings.warn('Got a request for an SQLite connection using APSW, but I cannot find an sqlite3-compatible installed version of that module (or the module is not installed at all). Furthermore, the place where the database is sitting ("%s") is on a filesystem that does **not** seem to support locks. I\'m returning a stock connection and hopping for the best.' % (filename,)) warnings.warn('Got a request for an SQLite connection using APSW, but I cannot find an sqlite3-compatible installed version of that module (or the module is not installed at all). Furthermore, the place where the database is sitting ("%s") is on a filesystem that does **not** seem to support locks. I\'m returning a stock connection and hopping for the best.' % (filename,))
def __call__(self): def __call__(self):
from sqlite3 import connect from sqlite3 import connect
...@@ -101,6 +103,7 @@ class SQLiteConnector(object): ...@@ -101,6 +103,7 @@ class SQLiteConnector(object):
return connect(self.filename, check_same_thread=False) return connect(self.filename, check_same_thread=False)
def create_engine(self, echo=False): def create_engine(self, echo=False):
"""Returns an SQLAlchemy engine""" """Returns an SQLAlchemy engine"""
...@@ -108,6 +111,7 @@ class SQLiteConnector(object): ...@@ -108,6 +111,7 @@ class SQLiteConnector(object):
from sqlalchemy.pool import NullPool from sqlalchemy.pool import NullPool
return create_engine('sqlite://', creator=self, echo=echo, poolclass=NullPool) return create_engine('sqlite://', creator=self, echo=echo, poolclass=NullPool)
def session(self, echo=False): def session(self, echo=False):
"""Returns an SQLAlchemy session""" """Returns an SQLAlchemy session"""
...@@ -115,6 +119,7 @@ class SQLiteConnector(object): ...@@ -115,6 +119,7 @@ class SQLiteConnector(object):
Session = sessionmaker(bind=self.create_engine(echo)) Session = sessionmaker(bind=self.create_engine(echo))
return Session() return Session()
def session(dbtype, dbfile, echo=False): def session(dbtype, dbfile, echo=False):
"""Creates a session to an SQLite database""" """Creates a session to an SQLite database"""
...@@ -127,12 +132,17 @@ def session(dbtype, dbfile, echo=False): ...@@ -127,12 +132,17 @@ def session(dbtype, dbfile, echo=False):
return Session() return Session()
def session_try_readonly(dbtype, dbfile, echo=False): def session_try_readonly(dbtype, dbfile, echo=False):
"""Creates a read-only session to an SQLite database. If read-only sessions """Creates a read-only session to an SQLite database.
are not supported by the underlying sqlite3 python DB driver, then a normal
session is returned. A warning is emitted in case the underlying filesystem If read-only sessions are not supported by the underlying sqlite3 python DB
does not support locking properly. driver, then a normal session is returned. A warning is emitted in case the
underlying filesystem does not support locking properly.
Raises:
NotImplementedError: if the dbtype is not supported.
Raises a NotImplementedError if the dbtype is not supported.
""" """
if dbtype != 'sqlite': if dbtype != 'sqlite':
...@@ -141,13 +151,19 @@ def session_try_readonly(dbtype, dbfile, echo=False): ...@@ -141,13 +151,19 @@ def session_try_readonly(dbtype, dbfile, echo=False):
connector = SQLiteConnector(dbfile, readonly=True, lock='unix-none') connector = SQLiteConnector(dbfile, readonly=True, lock='unix-none')
return connector.session(echo=echo) return connector.session(echo=echo)
def create_engine_try_nolock(dbtype, dbfile, echo=False): def create_engine_try_nolock(dbtype, dbfile, echo=False):
"""Creates an engine connected to an SQLite database with no locks. If """Creates an engine connected to an SQLite database with no locks.
engines without locks are not supported by the underlying sqlite3 python DB
driver, then a normal engine is returned. A warning is emitted if the If engines without locks are not supported by the underlying sqlite3 python
DB driver, then a normal engine is returned. A warning is emitted if the
underlying filesystem does not support locking properly in this case. underlying filesystem does not support locking properly in this case.
Raises a NotImplementedError if the dbtype is not supported.
Raises:
NotImplementedError: if the dbtype is not supported.
""" """
if dbtype != 'sqlite': if dbtype != 'sqlite':
...@@ -156,13 +172,19 @@ def create_engine_try_nolock(dbtype, dbfile, echo=False): ...@@ -156,13 +172,19 @@ def create_engine_try_nolock(dbtype, dbfile, echo=False):
connector = SQLiteConnector(dbfile, lock='unix-none') connector = SQLiteConnector(dbfile, lock='unix-none')
return connector.create_engine(echo=echo) return connector.create_engine(echo=echo)
def session_try_nolock(dbtype, dbfile, echo=False): def session_try_nolock(dbtype, dbfile, echo=False):
"""Creates a session to an SQLite database with no locks. If sessions without """Creates a session to an SQLite database with no locks.
locks are not supported by the underlying sqlite3 python DB driver, then a
normal session is returned. A warning is emitted if the underlying filesystem If sessions without locks are not supported by the underlying sqlite3 python
does not support locking properly in this case. DB driver, then a normal session is returned. A warning is emitted if the
underlying filesystem does not support locking properly in this case.
Raises:
NotImplementedError: if the dbtype is not supported.
Raises a NotImplementedError if the dbtype is not supported.
""" """
if dbtype != 'sqlite': if dbtype != 'sqlite':
...@@ -171,16 +193,17 @@ def session_try_nolock(dbtype, dbfile, echo=False): ...@@ -171,16 +193,17 @@ def session_try_nolock(dbtype, dbfile, echo=False):
connector = SQLiteConnector(dbfile, lock='unix-none') connector = SQLiteConnector(dbfile, lock='unix-none')
return connector.session(echo=echo) return connector.session(echo=echo)
def connection_string(dbtype, dbfile, opts={}): def connection_string(dbtype, dbfile, opts={}):
"""Returns a connection string for supported platforms """Returns a connection string for supported platforms
Keyword parameters Parameters:
dbtype (str): The type of database (only ``sqlite`` is supported for the
time being)
dbtype dbfile (str): The location of the file to be used
The type of database (only 'sqlite' is supported for the time being)
dbfile
The location of the file to be used
""" """
from sqlalchemy.engine.url import URL from sqlalchemy.engine.url import URL
......
# Not available in Python 2.7, but ok in Python 3.x # Not available in Python 2.7, but ok in Python 3.x
py:exc ValueError py:exc ValueError
py:exc NotImplementedError
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment