diff --git a/bob/devtools/bootstrap.py b/bob/devtools/bootstrap.py index 3ce36344fc16b0e24de1e4a5a224f4bf291fe813..73712a948b60098f559ef2200a204f1149a9cf07 100644 --- a/bob/devtools/bootstrap.py +++ b/bob/devtools/bootstrap.py @@ -318,21 +318,30 @@ def get_channels(public, stable, server, intranet, group): ) channels = [] + channels_dict = {} # do not use '/public' urls for public channels prefix = "/software/" + group if not stable: channels += [server + prefix + "/conda/label/beta"] # allowed betas + channels_dict["public/beta"] = channels[-1] channels += [server + prefix + "/conda"] + channels_dict["public/stable"] = channels[-1] if not public: prefix = "/private" if not stable: # allowed private channels channels += [server + prefix + "/conda/label/beta"] # allowed betas + channels_dict["private/beta"] = channels[-1] channels += [server + prefix + "/conda"] + channels_dict["private/stable"] = channels[-1] - return channels + upload_channel = channels_dict[ + f"{'public' if public else 'private'}/{'stable' if stable else 'beta'}" + ] + + return channels, upload_channel def setup_logger(logger, level): @@ -447,7 +456,8 @@ if __name__ == "__main__": # context. The URL should NOT work outside of Idiap's network. f.write( _BASE_CONDARC.replace( - "https://repo.anaconda.com/pkgs/main", "http://www.idiap.ch/defaults", + "https://repo.anaconda.com/pkgs/main", + "http://www.idiap.ch/defaults", ) ) @@ -496,7 +506,7 @@ if __name__ == "__main__": ) conda_bld_path = os.path.join(args.conda_root, "conda-bld") run_cmdline([conda_bin, "index", conda_bld_path]) - channels = get_channels( + channels, _ = get_channels( public=True, stable=True, server=_SERVER, intranet=True, group="bob" ) + ["defaults"] channels = ( @@ -515,13 +525,16 @@ if __name__ == "__main__": elif args.command == "channel": # installs from channel - channels = get_channels( - public=True, - stable=(args.tag is not None), - server=_SERVER, - intranet=True, - group="bob", - ) + ["defaults"] + channels, _ = ( + get_channels( + public=True, + stable=(args.tag is not None), + server=_SERVER, + intranet=True, + group="bob", + ) + + ["defaults"] + ) channels = ["--override-channels"] + ["--channel=%s" % k for k in channels] conda_cmd = "install" if args.envname in ("base", "root") else "create" cmd = ( diff --git a/bob/devtools/build.py b/bob/devtools/build.py index de2c378b2bb0e0a20209fb5dd388606228f2a9e3..bcea0410e0bae75bc83291506eef97fa6c303d71 100644 --- a/bob/devtools/build.py +++ b/bob/devtools/build.py @@ -576,7 +576,7 @@ def base_build( """ # if you get to this point, tries to build the package - channels = bootstrap.get_channels( + channels, upload_channel = bootstrap.get_channels( public=True, stable=True, server=server, intranet=intranet, group=group ) @@ -599,7 +599,7 @@ def base_build( return paths = get_output_path(metadata, conda_config) - urls = [exists_on_channel(channels[0], os.path.basename(k)) for k in paths] + urls = [exists_on_channel(upload_channel, os.path.basename(k)) for k in paths] if all(urls): logger.info( @@ -757,7 +757,7 @@ if __name__ == "__main__": ) public = args.visibility == "public" - channels = bootstrap.get_channels( + channels, upload_channel = bootstrap.get_channels( public=public, stable=(not is_prerelease), server=server, @@ -793,7 +793,7 @@ if __name__ == "__main__": # retrieve the current build number(s) for this build build_numbers = [ - next_build_number(channels[0], os.path.basename(k))[0] for k in paths + next_build_number(upload_channel, os.path.basename(k))[0] for k in paths ] # homogenize to the largest build number diff --git a/bob/devtools/scripts/build.py b/bob/devtools/scripts/build.py index be6518fd489b0eaebff4a40cd09963102043f888..23a0fc5ff3580cf65785ef750c79c060bc8a63d2 100644 --- a/bob/devtools/scripts/build.py +++ b/bob/devtools/scripts/build.py @@ -189,7 +189,7 @@ def build( # use default condarc_options = yaml.load(BASE_CONDARC, Loader=yaml.FullLoader) - channels = get_channels( + channels, upload_channel = get_channels( public=(not private), stable=stable, server=server, intranet=ci, group=group, ) @@ -201,7 +201,7 @@ def build( "\n - ".join(condarc_options["channels"]), ) - logger.info("Uploading resulting package to: %s", channels[0]) + logger.info("Uploading resulting package to: %s", upload_channel) # dump packages at base environment prefix = get_env_directory(os.environ["CONDA_EXE"], "base") @@ -246,7 +246,7 @@ def build( path = get_output_path(metadata, conda_config)[0] # gets the next build number - build_number, _ = next_build_number(channels[0], os.path.basename(path)) + build_number, _ = next_build_number(upload_channel, os.path.basename(path)) logger.info( "Building %s-%s-py%s (build: %d) for %s", diff --git a/bob/devtools/scripts/create.py b/bob/devtools/scripts/create.py index 33d851dbf58099712238c70f0b5a2077145df707..7981287ba30e21bff07b09649f98b3f97f32a4ee 100644 --- a/bob/devtools/scripts/create.py +++ b/bob/devtools/scripts/create.py @@ -212,7 +212,7 @@ def create( if "channels" not in condarc_options: from ..bootstrap import get_channels - channels = get_channels( + channels, _ = get_channels( public=(not private), stable=stable, server=server, diff --git a/bob/devtools/scripts/graph.py b/bob/devtools/scripts/graph.py index 641b3bd784023fab02c5b2911a339569e404ffb0..be4cddc0be747e95b9a43833c4caac8d94184b73 100644 --- a/bob/devtools/scripts/graph.py +++ b/bob/devtools/scripts/graph.py @@ -166,7 +166,7 @@ def graph( gl = get_gitlab_instance() # get potential channel upload and other auxiliary channels - channels = get_channels( + channels, upload_channel = get_channels( public=(not private), stable=stable, server=server, @@ -203,7 +203,7 @@ def graph( set_environment("NOSE_EVAL_ATTR", "") adj_matrix = compute_adjencence_matrix( - gl, package, conda_config, channels[0], deptypes=deptypes + gl, package, conda_config, upload_channel, deptypes=deptypes ) graph = generate_graph(adj_matrix, deptypes=deptypes, whitelist=whitelist) diff --git a/bob/devtools/scripts/local.py b/bob/devtools/scripts/local.py index b8565e9683c9c47040392a4078cdc229e7cf6fb0..33fd45a4e87d7eaa843a360e2f2b119f2083b70e 100644 --- a/bob/devtools/scripts/local.py +++ b/bob/devtools/scripts/local.py @@ -17,12 +17,24 @@ from . import ci logger = get_logger(__name__) -def set_up_environment_variables( - python, name_space, project_dir=".", project_visibility="public" -): +def set_up_environment_variables(python, name_space, project_dir): """This function sets up the proper environment variables when user wants to run the commands usually run on ci locally.""" - os.environ["CI_JOB_TOKEN"] = gitlab.Gitlab.from_config("idiap").private_token + project_dir = os.path.abspath(project_dir) + project_name = os.path.basename(project_dir) + gl = gitlab.Gitlab.from_config("idiap") + package_name = f"{name_space}/{project_name}" + logger.info("Inferred this package name: %s", package_name) + try: + use_package = gl.projects.get(package_name) + except Exception: + print( + f"Could not find {package_name} on Gitlab! " + "Did you specify the correct group?" + ) + raise + project_visibility = use_package.attributes["visibility"] + os.environ["CI_JOB_TOKEN"] = gl.private_token os.environ["CI_PROJECT_DIR"] = project_dir os.environ["CI_PROJECT_NAMESPACE"] = name_space os.environ["CI_PROJECT_VISIBILITY"] = project_visibility @@ -97,7 +109,9 @@ def docs(ctx, requirement, dry_run, python, group): \b """ - set_up_environment_variables(python=python, name_space=group) + set_up_environment_variables( + python=python, name_space=group, project_dir=os.path.dirname(requirement) + ) ctx.invoke(ci.docs, requirement=requirement, dry_run=dry_run) @@ -145,7 +159,9 @@ Examples: @click.pass_context def build(ctx, dry_run, recipe_dir, python, group): """Run the CI build step locally.""" - set_up_environment_variables(python=python, name_space=group) + set_up_environment_variables( + python=python, name_space=group, project_dir=os.path.join(recipe_dir, "..") + ) ctx.invoke(ci.build, dry_run=dry_run, recipe_dir=recipe_dir) @@ -186,5 +202,7 @@ Examples: @click.pass_context def base_build(ctx, order, dry_run, python, group): """Run the CI build step locally.""" - set_up_environment_variables(python=python, name_space=group) + set_up_environment_variables( + python=python, name_space=group, project_dir=os.path.dirname(order) + ) ctx.invoke(ci.base_build, order=order, dry_run=dry_run, group=group) diff --git a/bob/devtools/scripts/rebuild.py b/bob/devtools/scripts/rebuild.py index d42c581a947c2e9f19b7581f546908adb12c2314..33e2c0d336eed17dbf0640ec285189ed57ded63b 100644 --- a/bob/devtools/scripts/rebuild.py +++ b/bob/devtools/scripts/rebuild.py @@ -173,7 +173,7 @@ def rebuild( ) # get potential channel upload and other auxiliary channels - channels = get_channels( + channels, upload_channel = get_channels( public=(not private), stable=stable, server=server, intranet=ci, group=group, ) @@ -193,7 +193,7 @@ def rebuild( "\n - ".join(condarc_options["channels"]), ) - logger.info("Uploading resulting package to: %s", channels[0]) + logger.info("Uploading resulting package to: %s", upload_channel) # dump packages at base environment prefix = get_env_directory(os.environ["CONDA_EXE"], "base") @@ -238,7 +238,7 @@ def rebuild( path = get_output_path(metadata, conda_config)[0] # Get the latest build number - build_number, existing = next_build_number(channels[0], os.path.basename(path)) + build_number, existing = next_build_number(upload_channel, os.path.basename(path)) should_build = True @@ -249,7 +249,7 @@ def rebuild( ) if not os.path.exists(os.path.dirname(destpath)): os.makedirs(os.path.dirname(destpath)) - src = channels[0] + existing[0] + src = upload_channel + existing[0] logger.info("Downloading %s -> %s", src, destpath) urllib.request.urlretrieve(src, destpath) diff --git a/bob/devtools/scripts/test.py b/bob/devtools/scripts/test.py index 24f5ff8bbb66988c76d34a914b080991fc557086..f12042563ad53105a83ae18bed549e679f3a447b 100644 --- a/bob/devtools/scripts/test.py +++ b/bob/devtools/scripts/test.py @@ -167,7 +167,7 @@ def test( if "channels" not in condarc_options: from ..bootstrap import get_channels - channels = get_channels( + channels, _ = get_channels( public=(not private), stable=stable, server=server, intranet=ci, group=group ) condarc_options["channels"] = channels + ["defaults"]