From 5d11ace6d1f44b10137f34df0004054ebc019c0c Mon Sep 17 00:00:00 2001
From: Tiago Freitas Pereira <tiagofrepereira@gmail.com>
Date: Mon, 21 Oct 2019 18:02:44 +0200
Subject: [PATCH] Created 2 commands to process gitlab logs
---
bob/devtools/pipelines.py | 50 ++++++++++++
bob/devtools/scripts/pipelines.py | 130 ++++++++++++++++++++++++++++++
conda/meta.yaml | 3 +
setup.py | 2 +
4 files changed, 185 insertions(+)
create mode 100644 bob/devtools/pipelines.py
create mode 100644 bob/devtools/scripts/pipelines.py
diff --git a/bob/devtools/pipelines.py b/bob/devtools/pipelines.py
new file mode 100644
index 00000000..29ecf8b5
--- /dev/null
+++ b/bob/devtools/pipelines.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Pipeline utilities"""
+
+from tabulate import tabulate
+import re
+from datetime import datetime
+
+
+def process_log(log):
+ """
+ Summarizes the execution time of a pipeline given its Job log
+ """
+
+ current_package = None
+ logs = dict()
+ dates = []
+ for l in log:
+
+ # Check which package are we
+ if len(re.findall("Building bob/[a-z]*", l)) > 0:
+ logs[current_package] = dates
+ dates = []
+
+ pattern = re.findall("Building bob/[a-z]*", l)[0]
+ current_package = l[9:-1]
+ continue
+
+ # Checking the date
+ date = re.findall(
+ "[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2} [0-9]{2,2}:[0-9]{2,2}:[0-9]{2,2}", l
+ )
+ if len(date) > 0:
+ # logs[date[0]]=current_package
+ dates.append(date[0])
+
+ ## Last log
+ if len(dates) > 0:
+ logs[current_package] = dates
+
+ table = []
+ for k in logs:
+ first = datetime.strptime(logs[k][0], "%Y-%m-%d %H:%M:%S")
+ last = datetime.strptime(logs[k][-1], "%Y-%m-%d %H:%M:%S")
+ delta = ((last - first).seconds) / 60
+
+ table.append([str(k), str(first), str(round(delta, 2)) + "m"])
+
+ print(tabulate(table))
diff --git a/bob/devtools/scripts/pipelines.py b/bob/devtools/scripts/pipelines.py
new file mode 100644
index 00000000..7d21bc06
--- /dev/null
+++ b/bob/devtools/scripts/pipelines.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+
+import os
+
+import click
+import gitlab
+import urllib
+
+from . import bdt
+from ..release import get_gitlab_instance
+
+from ..log import verbosity_option, get_logger, echo_normal, echo_warning
+from ..pipelines import process_log
+logger = get_logger(__name__)
+from tabulate import tabulate
+
+@click.command(
+ epilog="""
+Examples:
+
+ 1. Process all the jobs from a particular pipeline
+
+ $ bdt gitlab process-pipelines bob/bob.nightlies pipelines
+
+ 2. Process a particular job from a pipeline
+
+ $ bdt gitlab process-pipelines bob/bob.nightlies pipelines --job-id xxx
+
+"""
+)
+@click.argument("package")
+@click.argument("pipeline")
+@click.option('--job-id', default=None, help='A job id from a pipeline')
+@verbosity_option()
+@bdt.raise_on_error
+def process_pipelines(package, pipeline, job_id):
+ """Returns the last tag information on a given PACKAGE."""
+
+ if "/" not in package:
+ raise RuntimeError('PACKAGE should be specified as "group/name"')
+
+ gl = get_gitlab_instance()
+
+ # we lookup the gitlab package once
+ try:
+ project = gl.projects.get(package)
+ pipeline = project.pipelines.get(pipeline)
+
+ jobs = [j for j in pipeline.jobs.list()]
+ if job_id is not None:
+ jobs = [j for j in jobs if int(j.attributes["id"])==int(job_id)]
+
+
+ if(len(jobs) == 0 ):
+ print("Job %s not found in the pipeline %s. Use `bdt gitlab get-pipelines` to search " % (job_id, pipeline.attributes["id"]))
+
+ # Reading log
+ try:
+ for j in jobs:
+ print("Pipeline %s, Job %s" % (pipeline.attributes["id"], int(j.attributes["id"])))
+ web_url = j.attributes["web_url"] + "/raw"
+ log = str(urllib.request.urlopen(web_url).read()).split("\\n")
+ process_log(log)
+ except urllib.error.HTTPError as e:
+ logger.warn(
+ "Gitlab access error - Log %s can't be found" % web_url, package
+ )
+ echo_warning("%s: unknown" % (package,))
+
+ logger.info(
+ "Found gitlab project %s (id=%d)",
+ project.attributes["path_with_namespace"],
+ project.id,
+ )
+
+ pass
+ except gitlab.GitlabGetError as e:
+ logger.warn("Gitlab access error - package %s does not exist?", package)
+ echo_warning("%s: unknown" % (package,))
+
+
+@click.command(
+ epilog="""
+Examples:
+
+ 1. Get the most recent pipelines from a particular project wit their corresponding job numbers
+
+ $ bdt gitlab get-pipelines bob/bob.nightlies
+
+
+"""
+)
+@click.argument("package")
+@verbosity_option()
+@bdt.raise_on_error
+def get_pipelines(package):
+ """Returns the CI pipelines given a given PACKAGE."""
+
+ if "/" not in package:
+ raise RuntimeError('PACKAGE should be specified as "group/name"')
+
+ gl = get_gitlab_instance()
+
+ # we lookup the gitlab package once
+ try:
+ project = gl.projects.get(package)
+ logger.info(
+ "Found gitlab project %s (id=%d)",
+ project.attributes["path_with_namespace"],
+ project.id,
+ )
+
+ pipelines = project.pipelines.list()
+ description = [["Pipeline", "Branch", "Status", "Jobs"]]
+ for pipeline in pipelines:
+ jobs = [j.attributes["id"] for j in pipeline.jobs.list()]
+ description.append(
+ [
+ "%s" % pipeline.attributes["id"],
+ "%s" % pipeline.attributes["ref"],
+ "%s" % pipeline.attributes["status"],
+ "%s" % jobs,
+ ]
+ )
+ print("Jobs from project %s" % package)
+ print(tabulate(description))
+
+ except gitlab.GitlabGetError as e:
+ logger.warn("Gitlab access error - package %s does not exist?", package)
+ echo_warning("%s: unknown" % (package,))
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 71775eec..df6141cd 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -48,6 +48,7 @@ requirements:
- jinja2
- termcolor
- psutil
+ - tabulate
test:
requires:
@@ -101,6 +102,8 @@ test:
- bdt dav rmtree --help
- bdt dav clean-betas --help
- bdt dav upload --help
+ - bdt gitlab process-pipelines --help
+ - bdt gitlab get-pipelines --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
- if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
diff --git a/setup.py b/setup.py
index ab67bfde..1e729138 100644
--- a/setup.py
+++ b/setup.py
@@ -67,6 +67,8 @@ setup(
'jobs = bob.devtools.scripts.jobs:jobs',
'visibility = bob.devtools.scripts.visibility:visibility',
'getpath = bob.devtools.scripts.getpath:getpath',
+ 'process-pipelines = bob.devtools.scripts.pipelines:process_pipelines',
+ 'get-pipelines- = bob.devtools.scripts.pipelines:get_pipelines',
],
'bdt.ci.cli': [
--
GitLab