How-to-port-old-bob-database-interface-to-the-new-csv-format.md

@@ -8,8 +8,97 @@ Learn information about the database:
 - in which bob.bio or bob.pad packages it was used
 - How many protocols does it have
 
-# Create a create function
-Create a script that reproduces the list files.
+# Create the csv files
+We will use a script to automatically convert most databases.
+If you want more detailed csv files for your database, you have to create them by your own custom code.
+
+```sh
+# create a bob 8 environment where databases where working then.
+$ conda create -n bob8_py36 \
+-c https://www.idiap.ch/software/bob/conda \
+-c defaults \
+-c https://www.idiap.ch/software/bob/conda/label/archive \
+python=3.6 bob=8 bob.pad.face bob.db.replaymobile
+# you would install the relevant bob.bio, bob.pad, and bob.db package
+
+# now, save the script provided below in a `convert_pad_database.py` file and run:
+$ python convert_pad_database.py bob.pad.face.config.replay_mobile --output ~/temp/bob_data/datasets/pad-face-replay-mobile -vvv
+# untar and verify the just created tarfile
+# upload it to our bob servers using bdt (to be developed https://gitlab.idiap.ch/bob/bob.devtools/-/issues/64)
+```
+
+```python
+import os
+
+import click
+from bob.extension.scripts.click_helper import ConfigCommand
+from bob.extension.scripts.click_helper import ResourceOption
+from bob.extension.scripts.click_helper import log_parameters
+from bob.extension.scripts.click_helper import verbosity_option
+
+
+@click.command(cls=ConfigCommand, entry_point_group="bob.pad.config")
+@click.option("--database", cls=ResourceOption, entry_point_group="bob.pad.database")
+@click.option("--output", cls=ResourceOption)
+@click.option("--protocol", "protocols", cls=ResourceOption, multiple=True)
+@click.option(
+    "--group",
+    "groups",
+    cls=ResourceOption,
+    multiple=True,
+    default=["train", "dev", "eval"],
+)
+@verbosity_option(cls=ResourceOption)
+def convert_database(database, output, protocols, groups, verbose, **kwargs):
+    import csv
+    import tarfile
+    import logging
+
+    logger = logging.getLogger(__name__)
+    log_parameters(logger)
+
+    if not protocols:
+        try:
+            protocols = [p.name for p in database.db.protocols()]
+        except Exception as e:
+            click.echo(f"WARNING: failed to retrieve the list of protocols from the lower level db. The error was: {e}")
+            protocols = [database.protocol]
+
+    os.makedirs(output, exist_ok=True)
+
+    for proto in protocols:
+        click.echo(f"Creating list files for protocol: {proto}")
+        database.protocol = proto
+        for grp in groups:
+            all_files = database.all_files(groups=grp, flat=True)
+            path = os.path.join(output, proto, grp + ".csv")
+            os.makedirs(os.path.dirname(path), exist_ok=True)
+            with open(path, "w", newline="") as f:
+                fieldnames = ["filename", "client_id", "attack_type"]
+                writer = csv.DictWriter(f, fieldnames=fieldnames)
+                writer.writeheader()
+                for padfile in all_files:
+                    writer.writerow(
+                        {
+                            "filename": padfile.make_path(
+                                "", database.original_extension
+                            ),
+                            "client_id": padfile.client_id,
+                            "attack_type": padfile.attack_type,
+                        }
+                    )
+
+    # create the final tarball
+    path = f"{output}.tar.gz"
+    with tarfile.open(path, "w:gz") as tar:
+        tar.add(output, arcname=".")
+
+    click.echo(f"Created {path}")
+
+
+if __name__ == "__main__":
+    convert_database()
+```
 
 # Documentation
 Document the protocols, data format, metadata
\ No newline at end of file