Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • beat/beat.cmdline
1 result
Show changes
Commits on Source (3)
Hide whitespace changes
Inline Side-by-side
beat/cmdline/cache.py
View file @ 3cc73e6c
...@@ -53,16 +53,16 @@ logger = logging.getLogger(__name__) ...@@ -53,16 +53,16 @@ logger = logging.getLogger(__name__)
def get_paths(config): def get_paths(config):
def func(z):
func = lambda z: z.split('.', 1)[0] return z.split(".", 1)[0]
retval = [] retval = []
for dirname, _, files in os.walk(config.cache): for dirname, _, files in os.walk(config.cache):
files = fnmatch.filter(files, '*.data') #avoid index-only files files = fnmatch.filter(files, "*.data") # avoid index-only files
if not files: if not files:
continue continue
d = dirname.replace(config.cache, '').strip(os.sep) d = dirname.replace(config.cache, "").strip(os.sep)
retval += list(set([os.path.join(d, func(k)) for k in files])) retval += list(set([os.path.join(d, func(k)) for k in files]))
return retval return retval
...@@ -70,10 +70,16 @@ def get_paths(config): ...@@ -70,10 +70,16 @@ def get_paths(config):
@click.group(cls=AliasedGroup) @click.group(cls=AliasedGroup)
@click.pass_context @click.pass_context
@click.option('--start', type=click.INT, help='If set, allows the user to ' @click.option(
'print only a few bits of the file') "--start",
@click.option('--end', type=click.INT, help='If set, allows the user to ' type=click.INT,
'print only a few bits of the file') help="If set, allows the user to " "print only a few bits of the file",
)
@click.option(
"--end",
type=click.INT,
help="If set, allows the user to " "print only a few bits of the file",
)
def cache(ctx, start, end): def cache(ctx, start, end):
"""Configuration manipulation and display""" """Configuration manipulation and display"""
pass pass
...@@ -82,117 +88,154 @@ def cache(ctx, start, end): ...@@ -82,117 +88,154 @@ def cache(ctx, start, end):
@cache.command() @cache.command()
@click.pass_context @click.pass_context
def clear(ctx): def clear(ctx):
'''Deletes all available cache """Deletes all available cache
To clear all available cache: To clear all available cache:
$ %(prog)s cache clear $ %(prog)s cache clear
''' """
import shutil import shutil
if os.path.isdir(ctx.meta['config'].cache):
for k in os.listdir(ctx.meta['config'].cache): if os.path.isdir(ctx.meta["config"].cache):
p = os.path.join(ctx.meta['config'].cache, k) for k in os.listdir(ctx.meta["config"].cache):
p = os.path.join(ctx.meta["config"].cache, k)
shutil.rmtree(p) shutil.rmtree(p)
@cache.command() @cache.command()
@click.argument('paths', nargs=-1, type=click.Path(exists=True)) @click.argument("paths", nargs=-1, type=click.Path(exists=True))
@click.pass_context @click.pass_context
@click.option('--sizes', help='If set, also print the size in bytes for ' @click.option(
'objects in a file. This triggers the full file readout', "--sizes",
is_flag=True) help="If set, also print the size in bytes for "
"objects in a file. This triggers the full file readout",
is_flag=True,
)
def info(ctx, paths, sizes): def info(ctx, paths, sizes):
'''Displays information about a particular cache file """Displays information about a particular cache file
To collect information about a particular cache file: To collect information about a particular cache file:
$ %(prog)s cache info 7f/d8/8d/a11178ac27075feaba8131fe878d6e3... $ %(prog)s cache info 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
''' """
config = ctx.meta['config'] config = ctx.meta["config"]
index_start = int(ctx.meta['start']) if 'start' in ctx.meta else None index_start = int(ctx.meta["start"]) if "start" in ctx.meta else None
index_end = int(ctx.meta['end']) if 'end' in ctx.meta else None index_end = int(ctx.meta["end"]) if "end" in ctx.meta else None
if not paths: if not paths:
paths = get_paths(config) paths = get_paths(config)
for path in paths: for path in paths:
logger.info('path: %s', path) logger.info("path: %s", path)
fullpath = os.path.join(config.cache, path + '.data') fullpath = os.path.join(config.cache, path + ".data")
f = CachedDataSource() f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end) status = f.setup(fullpath, config.path, index_start, index_end)
if not status: if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'", logger.error(
fullpath, config.path) "cannot setup data source with `%s' and prefix `%s'",
fullpath,
config.path,
)
return 1 return 1
logger.info(' dataformat: %s', f.dataformat.name) logger.info(" dataformat: %s", f.dataformat.name)
if sizes: if sizes:
counter = 0 counter = 0
logger.info(' index:') logger.info(" index:")
for data, start, end in f: for data, start, end in f:
size = len(data.pack()) size = len(data.pack())
counter += size counter += size
if start == end: if start == end:
logger.info(' [%d] - %d bytes', start, size) logger.info(" [%d] - %d bytes", start, size)
else: else:
logger.info(' [%d:%d] - %d bytes', start, end, size) logger.info(" [%d:%d] - %d bytes", start, end, size)
logger.info(' total (stripped-down) size: %d bytes', counter) logger.info(" total (stripped-down) size: %d bytes", counter)
else: else:
index = load_data_index(config.cache, path + '.data') index = load_data_index(config.cache, path + ".data")
logger.info(' objects : %d', len(index)-1) logger.info(" objects : %d", len(index) - 1)
@cache.command() @cache.command()
@click.argument('paths', nargs=-1) @click.argument("paths", nargs=-1)
@click.pass_context @click.pass_context
@raise_on_error @raise_on_error
def view(ctx, paths): def view(ctx, paths):
'''Displays information about a particular cache file """Displays information about a particular cache file
To view a particular cache file: To view a particular cache file:
$ %(prog)s cache view 7f/d8/8d/a11178ac27075feaba8131fe878d6e3... $ %(prog)s cache view 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
''' """
config = ctx.meta['config'] config = ctx.meta["config"]
index_start = int(ctx.meta['start']) if 'start' in ctx.meta else None index_start = int(ctx.meta["start"]) if "start" in ctx.meta else None
index_end = int(ctx.meta['end']) if 'end' in ctx.meta else None index_end = int(ctx.meta["end"]) if "end" in ctx.meta else None
if not paths: if not paths:
paths = get_paths(config) paths = get_paths(config)
for path in paths: for path in paths:
logger.info('path: %s', path) logger.info("path: %s", path)
fullpath = os.path.join(config.cache, path + '.data') fullpath = os.path.join(config.cache, path + ".data")
f = CachedDataSource() f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end) status = f.setup(fullpath, config.path, index_start, index_end)
if not status: if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'", logger.error(
fullpath, config.path) "cannot setup data source with `%s' and prefix `%s'",
fullpath,
config.path,
)
return 1 return 1
logger.info(' dataformat: %s', f.dataformat.name) logger.info(" dataformat: %s", f.dataformat.name)
for data, start, end in f: for data, start, end in f:
logger.extra(80 * '-') logger.extra(80 * "-")
if start == end: if start == end:
header = '[%d]: ' % start header = "[%d]: " % start
else: else:
header = '[%d:%d]: ' % (start, end) header = "[%d:%d]: " % (start, end)
json_data = data.as_dict() json_data = data.as_dict()
for name, value in json_data.items(): for name, value in json_data.items():
json_data[name] = common.stringify(value) json_data[name] = common.stringify(value)
json_data = simplejson.dumps( json_data = (
json_data, indent=2, simplejson.dumps(json_data, indent=2, cls=NumpyJSONEncoder)
cls=NumpyJSONEncoder).\ .replace('"BEAT_LIST_DELIMITER[', "[")
replace('"BEAT_LIST_DELIMITER[', '[')\ .replace(']BEAT_LIST_DELIMITER"', "]")
.replace(']BEAT_LIST_DELIMITER"', ']')\ .replace('"...",', "...")
.replace('"...",', '...')\ .replace('"BEAT_LIST_SIZE(', "(")
.replace('"BEAT_LIST_SIZE(', '(')\ .replace(')BEAT_LIST_SIZE"', ")")
.replace(')BEAT_LIST_SIZE"', ')') )
logger.info(header + json_data) logger.info(header + json_data)
@cache.command()
@click.option("--no-inputs", is_flag=True, default=False)
@click.argument("paths", nargs=-1, required=True)
@click.pass_context
def remove(ctx, paths, no_inputs):
"""Remove content of the cache entries passed in parameters
To remove an entry:
$ %(prog)s cache remove 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
"""
config = ctx.meta["config"]
for path in paths:
fullpath = os.path.join(config.cache, path[: path.rfind("/")])
file_list = [os.path.join(fullpath, file_) for file_ in os.listdir(fullpath)]
if file_list:
click.echo("About to delete:\n{}".format("\n".join(file_list)))
if no_inputs or click.confirm("Do you confirm the deletion ?"):
for file_ in file_list:
os.remove(file_)
else:
click.echo("Nothing to delete")
click.echo("Done")
beat/cmdline/test/test_cache.py
View file @ 3cc73e6c
...@@ -55,13 +55,15 @@ def call(*args, **kwargs): ...@@ -55,13 +55,15 @@ def call(*args, **kwargs):
use_prefix = kwargs.get("prefix", prefix) use_prefix = kwargs.get("prefix", prefix)
arguments = ["--prefix", use_prefix, "--cache", tmp_prefix] + list(args)
verbose = kwargs.get("verbose", False)
if not verbose:
arguments.insert(0, "--test-mode")
runner = CliRunner() runner = CliRunner()
with runner.isolated_filesystem(): with runner.isolated_filesystem():
result = runner.invoke( result = runner.invoke(main_cli.main, arguments, catch_exceptions=False)
main_cli.main,
["--test-mode", "--prefix", use_prefix, "--cache", tmp_prefix] + list(args),
catch_exceptions=False,
)
return result.exit_code, result.output return result.exit_code, result.output
...@@ -97,6 +99,19 @@ def test_cache_view(): ...@@ -97,6 +99,19 @@ def test_cache_view():
nose.tools.eq_(ex_code, 0, out) nose.tools.eq_(ex_code, 0, out)
@slow
def test_cache_remove():
nose.tools.assert_not_equal(len(os.listdir(tmp_prefix)), 0)
ex_code, out = call("cache", "info", verbose=True)
nose.tools.eq_(ex_code, 0, out)
entry = out.split("\n")[0]
entry = entry[6:]
ex_code, out = call("cache", "remove", "--no-inputs", entry)
nose.tools.eq_(ex_code, 0, out)
nose.tools.assert_true(out.startswith("About to delete:"))
nose.tools.assert_true(out.endswith("Done\n"))
@slow @slow
def test_cache_clear(): def test_cache_clear():
nose.tools.assert_not_equal(len(os.listdir(tmp_prefix)), 0) nose.tools.assert_not_equal(len(os.listdir(tmp_prefix)), 0)
......