diff --git a/bob/db/base/driver.py b/bob/db/base/driver.py
index 8842aa19532f7cad9c02e298cd8882b034f1f102..4b197620263686a19566974fab1a74099698107c 100644
--- a/bob/db/base/driver.py
+++ b/bob/db/base/driver.py
@@ -61,22 +61,47 @@ def dbshell_command(subparsers):
def upload(arguments):
- """For SQLite databases: uploads the db.sql3 database file to a server."""
- # get the file name of the target db
- assert len(arguments.files) == 1
- assert os.path.basename(arguments.files[0]) == 'db.sql3'
- source_file = arguments.files[0]
- target_file = os.path.join(arguments.destination, arguments.name + ".tar.bz2")
-
- if os.path.exists(source_file):
- print ("Compressing file '%s' to '%s'" %(source_file, target_file))
- import tarfile, stat
- f = tarfile.open(target_file, 'w:bz2')
- f.add(source_file, os.path.basename(source_file))
- f.close()
- os.chmod(target_file, stat.S_IRUSR|stat.S_IWUSR | stat.S_IRGRP|stat.S_IWGRP | stat.S_IROTH)
- else:
- print ("WARNING! Database file '%s' is not available. Did you run 'bob_dbmanage %s create' ?" % (source_file, arguments.name))
+ """Uploads generated metadata to the Idiap build server"""
+
+ import pkg_resources
+ basedir = pkg_resources.resource_filename('bob.db.%s' % arguments.name, '')
+ assert basedir, "Database and package names do not match. Your declared " \
+ "database name should be <name>, if your package is called bob.db.<name>"
+
+ target_file = os.path.join(arguments.destination,
+ arguments.name + ".tar.bz2")
+
+ # check all files exist
+ for p in arguments.files:
+ if not os.path.exists(p):
+ raise IOError("Metadata file `%s' is not available. Did you run " \
+ "`create' before attempting to upload?" % (p,))
+
+ # if destination exists, try to erase it before
+ if os.path.exists(target_file):
+ try:
+ os.unlink(target_file)
+ except Exception as e:
+ print("Cannot erase existing file `%s': %s" % (target_file, e))
+
+ # if you get here, all files are there, ready to package
+ print("Compressing metadata files to `%s'" % (target_file,))
+
+ # compress
+ import tarfile
+
+ f = tarfile.open(target_file, 'w:bz2')
+ for k,p in enumerate(arguments.files):
+ n = os.path.relpath(p, basedir)
+ print("+ [%d/%d] %s" % (k+1, len(arguments.files), n))
+ f.add(p, n)
+ f.close()
+
+ # set permissions for sane Idiap storage
+ import stat
+ perms = stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IWGRP|stat.S_IROTH
+ os.chmod(target_file, perms)
+
def upload_command(subparsers):
"""Adds a new 'upload' subcommand to your parser"""
@@ -89,41 +114,81 @@ def upload_command(subparsers):
def download(arguments):
- """For SQLite databases: Downloads the db.sql3 database file from a server."""
- # get the file name of the target db
- assert len(arguments.files) == 1
- assert os.path.basename(arguments.files[0]) == 'db.sql3'
- target_file = arguments.files[0]
- if os.path.exists(target_file) and not arguments.force:
- print ("Skipping download of file '%s' since it exists already." % target_file)
- else:
- # get URL of database file
- source_url = os.path.join(arguments.source, arguments.name + ".tar.bz2")
- # download
- import sys, tempfile, tarfile
- if sys.version_info[0] <= 2:
- import urllib2 as urllib
- else:
- import urllib.request as urllib
+ """Downloads and uncompresses meta data generated files from Idiap
+ Parameters:
+
+ arguments (argparse.Namespace): A set of arguments passed by the
+ command-line parser
+
+
+ Returns:
+
+ int: A POSIX compliant return value of ``0`` if the download is successful,
+ or ``1`` in case it is not.
+
+ """
+
+ # check all files don't exist
+ for p in arguments.files:
+ if os.path.exists(p):
+ if arguments.force:
+ os.unlink(p)
+ else:
+ raise IOError("Metadata file `%s' is already available. Please " \
+ "remove self-generated files before attempting download or " \
+ "--force" % (p,))
+
+ # if you get here, all files aren't there, unpack
+ source_url = os.path.join(arguments.source, arguments.name + ".tar.bz2")
+
+ target_dir = arguments.test_dir #test case
+
+ if not target_dir: #tries to dig it up
+
+ import pkg_resources
try:
- print ("Extracting url '%s' to '%s'" %(source_url, target_file))
- u = urllib.urlopen(source_url)
- f = tempfile.NamedTemporaryFile(suffix = ".tar.bz2")
- open(f.name, 'wb').write(u.read())
- t = tarfile.open(fileobj=f, mode = 'r:bz2')
- t.extract(os.path.basename(target_file), os.path.dirname(target_file))
- t.close()
- f.close()
- return False
- except Exception as e:
- print ("Error while downloading: '%s'" % e)
- return True
+ target_dir = pkg_resources.resource_filename('bob.db.%s' % \
+ arguments.name, '')
+ except ImportError as e:
+ print("The package `bob.db.%s' is not currently installed" % \
+ (arguments.name,))
+ print("N.B.: The database and package names **must** match. Your " \
+ "package should be named `bob.db.%s', if the driver name for your "
+ "database is `<name>'")
+ return 1
+
+ # download file from Idiap server, unpack and remove it
+ import sys
+ import tempfile
+ import tarfile
+ import pkg_resources
+ if sys.version_info[0] <= 2:
+ import urllib2 as urllib
+ else:
+ import urllib.request as urllib
+
+ try:
+ print ("Extracting url `%s' into `%s'" %(source_url, target_dir))
+ u = urllib.urlopen(source_url)
+ f = tempfile.NamedTemporaryFile(suffix = ".tar.bz2")
+ open(f.name, 'wb').write(u.read())
+ t = tarfile.open(fileobj=f, mode='r:bz2')
+ t.extractall(target_dir)
+ t.close()
+ f.close()
+ return 0
+
+ except Exception as e:
+ print ("Error while downloading: %s" % e)
+ return 1
def download_command(subparsers):
"""Adds a new 'download' subcommand to your parser"""
+ from argparse import SUPPRESS
+
if 'DOCSERVER' in os.environ:
USE_SERVER=os.environ['DOCSERVER']
else:
@@ -133,6 +198,7 @@ def download_command(subparsers):
parser.add_argument("--source",
default="%s/software/bob/databases/latest/" % USE_SERVER)
parser.add_argument("--force", action='store_true', help = "Overwrite existing database files?")
+ parser.add_argument("--test-dir", help=SUPPRESS)
parser.set_defaults(func=download)
return parser
@@ -174,58 +240,86 @@ def version_command(subparsers):
@six.add_metaclass(abc.ABCMeta)
class Interface(object):
- """Base manager for Bob databases"""
+ """Base manager for Bob databases
+
+ You should derive and implement an Interface object on every ``bob.db``
+ package you create.
+ """
@abc.abstractmethod
def name(self):
- '''Returns a simple name for this database, w/o funny characters, spaces'''
+ '''The name of this database
+
+ Returns:
+
+ str: a Python-conforming name for this database. This **must** match the
+ package name. If the package is named ``bob.db.foo``, then this function
+ must return ``foo``.
+ '''
+
return
@abc.abstractmethod
def files(self):
- '''Returns a python iterable with all auxiliary files needed.
+ '''Files containing meta-data for this package
+
+ Returns:
+
+ list: A python iterable with all metadata files needed. The paths listed
+ by this method should correspond to full paths (not relative ones) w.r.t.
+ the database package implementing it. This is normally achieved by using
+ ``pkg_resources.resource_filename()``.
- The values should be take w.r.t. where the python file that declares the
- database is sitting at.
'''
+
return
@abc.abstractmethod
def version(self):
- '''Returns the current version number defined in setup.py'''
+ '''The version of this package
+
+ Returns:
+
+ str: The current version number defined in ``setup.py``
+ '''
+
return
@abc.abstractmethod
def type(self):
- '''Returns the type of auxiliary files you have for this database
+ '''The type of auxiliary files you have for this database
- If you return 'sqlite', then we append special actions such as 'dbshell'
- on 'bob_dbmanage.py' automatically for you. Otherwise, we don't.
+ Returns:
- If you use auxiliary text files, just return 'text'. We may provide
- special services for those types in the future.
+ str: A string defining the type of database implemented. You can return
+ only two values on this function, either ``sqlite`` or ``text``. If you
+ return ``sqlite``, then we append special actions such as ``dbshell`` on
+ ``bob_dbmanage`` automatically for you. Otherwise, we don't.
- Use the special name 'builtin' if this database is an integral part of Bob.
'''
+
return
def setup_parser(self, parser, short_description, long_description):
'''Sets up the base parser for this database.
- Keyword arguments:
+ Parameters:
+
+ short_description (str): A short description (one-liner) for this
+ database
+
+ long_description (str): A more involved explanation of this database
- short_description
- A short description (one-liner) for this database
- long_description
- A more involved explanation of this database
+ Returns:
+
+ argparse.ArgumentParser: a subparser, ready so you can add commands on
- Returns a subparser, ready to be added commands on
'''
from argparse import RawDescriptionHelpFormatter
@@ -248,11 +342,13 @@ class Interface(object):
# adds some stock commands
version_command(subparsers)
- if type in ('sqlite',):
- dbshell_command(subparsers)
+ if files:
upload_command(subparsers)
download_command(subparsers)
+ if type in ('sqlite',):
+ dbshell_command(subparsers)
+
if files is not None:
files_command(subparsers)
@@ -261,7 +357,7 @@ class Interface(object):
@abc.abstractmethod
def add_commands(self, parser):
- '''Adds commands to a given (:py:mod:`argparse`) parser.
+ '''Adds commands to a given :py:class:`argparse.ArgumentParser`
This method, effectively, allows you to define special commands that your
database will be able to perform when called from the common driver like
@@ -269,16 +365,20 @@ class Interface(object):
You are not obliged to overwrite this method. If you do, you will have the
chance to establish your own commands. You don't have to worry about stock
- commands such as :py:meth:`files` or :py:meth:`version`. They will be automatically
- hooked-in depending on the values you return for :py:meth:`type` and
- :py:meth:`files`.
+ commands such as :py:meth:`files` or :py:meth:`version`. They will be
+ automatically hooked-in depending on the values you return for
+ :py:meth:`type` and :py:meth:`files`.
+
- Keyword arguments
+ Parameters:
+
+ parser (argparse.ArgumentParser): An instance of a parser that you can
+ customize, i.e., call :py:meth:`argparse.ArgumentParser.add_argument`
+ on.
- parser
- An instance of a :py:class:`argparse.ArgumentParser` that you can customize, i.e., call
- :py:meth:`argparse.ArgumentParser.add_argument` on.
'''
+
return
+
__all__ = ('Interface',)
diff --git a/bob/db/base/tests/test_driver.py b/bob/db/base/tests/test_driver.py
index 8a6e5b37304d0188eb1be4ee4dd76f2ac53aaaa0..281dc7b430dd1288538877916a1b74d5a908d329 100644
--- a/bob/db/base/tests/test_driver.py
+++ b/bob/db/base/tests/test_driver.py
@@ -18,12 +18,12 @@ def test_download_01():
tmpdir = tempfile.mkdtemp()
try:
arguments = Namespace(files=[tmpdir+'/db.sql3'], force=False,
- name='non_existing_db', type='sqlite', version='0.9.0',
+ name='does_not_exist', test_dir=tmpdir, version='0.9.0',
source='%s/software/bob/databases/latest/' % USE_SERVER)
- assert download(arguments) == True
+ assert download(arguments) == 1 #error #error #error #error
arguments = Namespace(files=[tmpdir+'/db.sql3'], force=False,
- name='biowave_test', type='sqlite', version='0.9.0',
+ name='banca', test_dir=tmpdir, version='0.9.0',
source='%s/software/bob/databases/latest/' % USE_SERVER)
- assert download(arguments) == False
+ assert download(arguments) == 0 #success
finally:
shutil.rmtree(tmpdir)
diff --git a/bob/db/base/utils.py b/bob/db/base/utils.py
index d3b43a712635014ceb7220f66cd6edc739da7259..b31410e8edac2e0a7e0a145e291a14711ec82a3e 100644
--- a/bob/db/base/utils.py
+++ b/bob/db/base/utils.py
@@ -8,6 +8,7 @@
import os
+
class null(object):
"""A look-alike stream that discards the input"""
@@ -21,6 +22,7 @@ class null(object):
pass
+
def apsw_is_available():
"""Checks lock-ability for SQLite on the current file system"""
@@ -39,8 +41,19 @@ def apsw_is_available():
# if you get to this point, all seems OK
return True
+
class SQLiteConnector(object):
- '''An object that handles the connection to SQLite databases.'''
+ '''An object that handles the connection to SQLite databases.
+
+ Parameters:
+
+ filename (str): The name of the file containing the SQLite database
+
+ readonly (bool): Should I try and open the database in read-only mode?
+
+ lock (str): Any vfs name as output by apsw.vfsnames()
+
+ '''
@staticmethod
def filesystem_is_lockable(database):
@@ -62,20 +75,8 @@ class SQLiteConnector(object):
APSW_IS_AVAILABLE = apsw_is_available()
- def __init__(self, filename, readonly=False, lock=None):
- """Initializes the connector
- Keyword arguments
-
- filename
- The name of the file containing the SQLite database
-
- readonly
- Should I try and open the database in read-only mode?
-
- lock
- Any vfs name as output by apsw.vfsnames()
- """
+ def __init__(self, filename, readonly=False, lock=None):
self.readonly = readonly
self.vfs = lock
@@ -87,6 +88,7 @@ class SQLiteConnector(object):
import warnings
warnings.warn('Got a request for an SQLite connection using APSW, but I cannot find an sqlite3-compatible installed version of that module (or the module is not installed at all). Furthermore, the place where the database is sitting ("%s") is on a filesystem that does **not** seem to support locks. I\'m returning a stock connection and hopping for the best.' % (filename,))
+
def __call__(self):
from sqlite3 import connect
@@ -101,6 +103,7 @@ class SQLiteConnector(object):
return connect(self.filename, check_same_thread=False)
+
def create_engine(self, echo=False):
"""Returns an SQLAlchemy engine"""
@@ -108,6 +111,7 @@ class SQLiteConnector(object):
from sqlalchemy.pool import NullPool
return create_engine('sqlite://', creator=self, echo=echo, poolclass=NullPool)
+
def session(self, echo=False):
"""Returns an SQLAlchemy session"""
@@ -115,6 +119,7 @@ class SQLiteConnector(object):
Session = sessionmaker(bind=self.create_engine(echo))
return Session()
+
def session(dbtype, dbfile, echo=False):
"""Creates a session to an SQLite database"""
@@ -127,12 +132,17 @@ def session(dbtype, dbfile, echo=False):
return Session()
def session_try_readonly(dbtype, dbfile, echo=False):
- """Creates a read-only session to an SQLite database. If read-only sessions
- are not supported by the underlying sqlite3 python DB driver, then a normal
- session is returned. A warning is emitted in case the underlying filesystem
- does not support locking properly.
+ """Creates a read-only session to an SQLite database.
+
+ If read-only sessions are not supported by the underlying sqlite3 python DB
+ driver, then a normal session is returned. A warning is emitted in case the
+ underlying filesystem does not support locking properly.
+
+
+ Raises:
+
+ NotImplementedError: if the dbtype is not supported.
- Raises a NotImplementedError if the dbtype is not supported.
"""
if dbtype != 'sqlite':
@@ -141,13 +151,19 @@ def session_try_readonly(dbtype, dbfile, echo=False):
connector = SQLiteConnector(dbfile, readonly=True, lock='unix-none')
return connector.session(echo=echo)
+
def create_engine_try_nolock(dbtype, dbfile, echo=False):
- """Creates an engine connected to an SQLite database with no locks. If
- engines without locks are not supported by the underlying sqlite3 python DB
- driver, then a normal engine is returned. A warning is emitted if the
+ """Creates an engine connected to an SQLite database with no locks.
+
+ If engines without locks are not supported by the underlying sqlite3 python
+ DB driver, then a normal engine is returned. A warning is emitted if the
underlying filesystem does not support locking properly in this case.
- Raises a NotImplementedError if the dbtype is not supported.
+
+ Raises:
+
+ NotImplementedError: if the dbtype is not supported.
+
"""
if dbtype != 'sqlite':
@@ -156,13 +172,19 @@ def create_engine_try_nolock(dbtype, dbfile, echo=False):
connector = SQLiteConnector(dbfile, lock='unix-none')
return connector.create_engine(echo=echo)
+
def session_try_nolock(dbtype, dbfile, echo=False):
- """Creates a session to an SQLite database with no locks. If sessions without
- locks are not supported by the underlying sqlite3 python DB driver, then a
- normal session is returned. A warning is emitted if the underlying filesystem
- does not support locking properly in this case.
+ """Creates a session to an SQLite database with no locks.
+
+ If sessions without locks are not supported by the underlying sqlite3 python
+ DB driver, then a normal session is returned. A warning is emitted if the
+ underlying filesystem does not support locking properly in this case.
+
+
+ Raises:
+
+ NotImplementedError: if the dbtype is not supported.
- Raises a NotImplementedError if the dbtype is not supported.
"""
if dbtype != 'sqlite':
@@ -171,16 +193,17 @@ def session_try_nolock(dbtype, dbfile, echo=False):
connector = SQLiteConnector(dbfile, lock='unix-none')
return connector.session(echo=echo)
+
def connection_string(dbtype, dbfile, opts={}):
"""Returns a connection string for supported platforms
- Keyword parameters
+ Parameters:
+
+ dbtype (str): The type of database (only ``sqlite`` is supported for the
+ time being)
- dbtype
- The type of database (only 'sqlite' is supported for the time being)
+ dbfile (str): The location of the file to be used
- dbfile
- The location of the file to be used
"""
from sqlalchemy.engine.url import URL
diff --git a/doc/nitpick-exceptions.txt b/doc/nitpick-exceptions.txt
index 4d428994e87705408041d46601d28dadb7c35b29..675e64439e0c139c826a04730353ac048735c482 100644
--- a/doc/nitpick-exceptions.txt
+++ b/doc/nitpick-exceptions.txt
@@ -1,2 +1,3 @@
# Not available in Python 2.7, but ok in Python 3.x
py:exc ValueError
+py:exc NotImplementedError