Commit 7f98a4cf authored by André Anjos's avatar André Anjos 💬
Browse files

Merge branch 'issue_1' into 'master'

Clean-up database download (closes #1)

See merge request !2
parents 9a17f58b 31aad2d9
Pipeline #5900 passed with stages
in 18 minutes and 23 seconds
......@@ -28,7 +28,7 @@ stages:
- ./_ci/install.sh _ci #updates
- ./_ci/before_build.sh
script:
- ./_ci/build.sh hci_tagging
- ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
artifacts:
......
......@@ -204,97 +204,28 @@ def checkfiles(args):
return 0
def _files():
filelist = pkg_resources.resource_filename(__name__, 'files.txt')
return [k.strip() for k in open(filelist, 'rt').readlines() if k.strip()]
def upload(arguments):
"""Uploads generated metadata to the Idiap build server"""
target_file = os.path.join(arguments.destination,
arguments.name + ".tar.bz2")
# check all files exist
names = _files()
paths = [pkg_resources.resource_filename(__name__, f) for f in names]
for n,p in zip(names, paths):
if not os.path.exists(p):
raise IOError("Metadata file `%s' (path: %s) is not available. Did you run `mkmeta' before attempting to upload?" % (n, p))
# if you get here, all files are there, ready to package
print("Compressing metadata files to `%s'" % (target_file,))
# compress
import tarfile
f = tarfile.open(target_file, 'w:bz2')
for k,(n,p) in enumerate(zip(names, paths)):
print("+ [%d/%d] %s" % (k+1, len(names), n))
f.add(p, n)
f.close()
# set permissions for sane Idiap storage
import stat
perms = stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IWGRP|stat.S_IROTH
os.chmod(target_file, perms)
def download(arguments):
"""Downloads and uncompresses meta data generated files from Idiap"""
# check all files don't exist
names = _files()
paths = [pkg_resources.resource_filename(__name__, f) for f in names]
for n,p in zip(names, paths):
if os.path.exists(p):
if arguments.force:
os.unlink(p)
else:
raise IOError("Metadata file `%s' (path: %s) is already available. Please remove self-generated files before attempting download or --force" % (n, p))
# if you get here, all files aren't there, unpack
source_url = os.path.join(arguments.source, arguments.name + ".tar.bz2")
# download file from Idiap server, unpack and remove it
import sys, tempfile, tarfile
if sys.version_info[0] <= 2:
import urllib2 as urllib
else:
import urllib.request as urllib
try:
print ("Extracting url `%s' (this may be long)..." %(source_url,))
u = urllib.urlopen(source_url)
f = tempfile.NamedTemporaryFile(suffix = ".tar.bz2")
open(f.name, 'wb').write(u.read())
t = tarfile.open(fileobj=f, mode='r:bz2')
t.extractall(pkg_resources.resource_filename(__name__, ''))
t.close()
f.close()
return False
except Exception as e:
print ("Error while downloading: %s" % e)
return True
class Interface(BaseInterface):
def name(self):
return 'hci_tagging'
def files(self):
from . import LOCATION
return [LOCATION]
basedir = pkg_resources.resource_filename(__name__, '')
filelist = os.path.join(basedir, 'files.txt')
return [os.path.join(basedir, k.strip()) for k in \
open(filelist, 'rt').readlines() if k.strip()]
def version(self):
import pkg_resources
return pkg_resources.require('bob.db.%s' % self.name())[0].version
def type(self):
return 'text'
def add_commands(self, parser):
"""Add specific subcommands that the action "dumplist" can use"""
......@@ -343,22 +274,3 @@ class Interface(BaseInterface):
debug_parser.add_argument('--limit', dest="limit", default=0, type=int, help="Limits the number of objects to treat (defaults to '%(default)')")
debug_parser.add_argument('--self-test', dest="selftest", default=False, action='store_true', help=SUPPRESS)
debug_parser.set_defaults(func=debug) #action
# add upload command
upload_meta = upload.__doc__
upload_parser = subparsers.add_parser('upload', help=upload.__doc__)
upload_parser.add_argument("--destination",
default="/idiap/group/torch5spro/databases/latest")
upload_parser.set_defaults(func=upload)
# add download command
if 'DOCSERVER' in os.environ: USE_SERVER=os.environ['DOCSERVER']
else: USE_SERVER='https://www.idiap.ch'
download_meta = download.__doc__
download_parser = subparsers.add_parser('download',
help=download.__doc__)
download_parser.add_argument("--source",
default="%s/software/bob/databases/latest/" % USE_SERVER)
download_parser.add_argument("--force", action='store_true',
help = "Overwrite existing metadata files?")
download_parser.set_defaults(func=download)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment