Clean-up database download (closes #1)

.gitlab-ci.yml

@@ -28,7 +28,7 @@ stages:
     - ./_ci/install.sh _ci #updates
     - ./_ci/before_build.sh
   script:
-    - ./_ci/build.sh hci_tagging
+    - ./_ci/build.sh
   after_script:
     - ./_ci/after_build.sh
   artifacts:

bob/db/hci_tagging/driver.py

@@ -204,97 +204,28 @@ def checkfiles(args):
   return 0
 
 
-def _files():
-  filelist = pkg_resources.resource_filename(__name__, 'files.txt')
-  return [k.strip() for k in open(filelist, 'rt').readlines() if k.strip()]
-
-
-def upload(arguments):
-  """Uploads generated metadata to the Idiap build server"""
-
-  target_file = os.path.join(arguments.destination,
-      arguments.name + ".tar.bz2")
-
-  # check all files exist
-  names = _files()
-  paths = [pkg_resources.resource_filename(__name__, f) for f in names]
-  for n,p in zip(names, paths):
-    if not os.path.exists(p):
-      raise IOError("Metadata file `%s' (path: %s) is not available. Did you run `mkmeta' before attempting to upload?" % (n, p))
-
-  # if you get here, all files are there, ready to package
-  print("Compressing metadata files to `%s'" % (target_file,))
-
-  # compress
-  import tarfile
-  f = tarfile.open(target_file, 'w:bz2')
-  for k,(n,p) in enumerate(zip(names, paths)):
-    print("+ [%d/%d] %s" % (k+1, len(names), n))
-    f.add(p, n)
-  f.close()
-
-  # set permissions for sane Idiap storage
-  import stat
-  perms = stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IWGRP|stat.S_IROTH
-  os.chmod(target_file, perms)
-
-
-def download(arguments):
-  """Downloads and uncompresses meta data generated files from Idiap"""
-
-  # check all files don't exist
-  names = _files()
-  paths = [pkg_resources.resource_filename(__name__, f) for f in names]
-  for n,p in zip(names, paths):
-    if os.path.exists(p):
-      if arguments.force:
-        os.unlink(p)
-      else:
-        raise IOError("Metadata file `%s' (path: %s) is already available. Please remove self-generated files before attempting download or --force" % (n, p))
-
-  # if you get here, all files aren't there, unpack
-  source_url = os.path.join(arguments.source, arguments.name + ".tar.bz2")
-
-  # download file from Idiap server, unpack and remove it
-  import sys, tempfile, tarfile
-  if sys.version_info[0] <= 2:
-    import urllib2 as urllib
-  else:
-    import urllib.request as urllib
-
-  try:
-    print ("Extracting url `%s' (this may be long)..." %(source_url,))
-    u = urllib.urlopen(source_url)
-    f = tempfile.NamedTemporaryFile(suffix = ".tar.bz2")
-    open(f.name, 'wb').write(u.read())
-    t = tarfile.open(fileobj=f, mode='r:bz2')
-    t.extractall(pkg_resources.resource_filename(__name__, ''))
-    t.close()
-    f.close()
-    return False
-
-  except Exception as e:
-    print ("Error while downloading: %s" % e)
-    return True
-
-
-
 class Interface(BaseInterface):
 
   def name(self):
     return 'hci_tagging'
 
+
   def files(self):
-    from . import LOCATION
-    return [LOCATION]
+    basedir = pkg_resources.resource_filename(__name__, '')
+    filelist = os.path.join(basedir, 'files.txt')
+    return [os.path.join(basedir, k.strip()) for k in \
+        open(filelist, 'rt').readlines() if k.strip()]
+
 
   def version(self):
     import pkg_resources
     return pkg_resources.require('bob.db.%s' % self.name())[0].version
 
+
   def type(self):
     return 'text'
 
+
   def add_commands(self, parser):
     """Add specific subcommands that the action "dumplist" can use"""
 
@@ -343,22 +274,3 @@ class Interface(BaseInterface):
     debug_parser.add_argument('--limit', dest="limit", default=0, type=int, help="Limits the number of objects to treat (defaults to '%(default)')")
     debug_parser.add_argument('--self-test', dest="selftest", default=False, action='store_true', help=SUPPRESS)
     debug_parser.set_defaults(func=debug) #action
-
-    # add upload command
-    upload_meta = upload.__doc__
-    upload_parser = subparsers.add_parser('upload', help=upload.__doc__)
-    upload_parser.add_argument("--destination",
-        default="/idiap/group/torch5spro/databases/latest")
-    upload_parser.set_defaults(func=upload)
-
-    # add download command
-    if 'DOCSERVER' in os.environ: USE_SERVER=os.environ['DOCSERVER']
-    else: USE_SERVER='https://www.idiap.ch'
-    download_meta = download.__doc__
-    download_parser = subparsers.add_parser('download',
-        help=download.__doc__)
-    download_parser.add_argument("--source",
-            default="%s/software/bob/databases/latest/" % USE_SERVER)
-    download_parser.add_argument("--force", action='store_true',
-        help = "Overwrite existing metadata files?")
-    download_parser.set_defaults(func=download)