Merge branch 'process-log' into 'master'

Created 2 commands to process gitlab logs

See merge request !124

bob/devtools/pipelines.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Pipeline utilities"""
+
+from tabulate import tabulate
+import re
+from datetime import datetime
+
+
+def process_log(log):
+    """
+    Summarizes the execution time of a pipeline given its Job log
+    """
+
+    current_package = None
+    logs = dict()
+    dates = []
+    for l in log:
+
+        # Check which package are we
+        if len(re.findall("Building bob/[a-z]*", l)) > 0:
+            logs[current_package] = dates
+            dates = []
+
+            pattern = re.findall("Building bob/[a-z]*", l)[0]
+            current_package = l[9:-1]
+            continue
+
+        # Checking the date
+        date = re.findall(
+            "[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2} [0-9]{2,2}:[0-9]{2,2}:[0-9]{2,2}", l
+        )
+        if len(date) > 0:
+            # logs[date[0]]=current_package
+            dates.append(date[0])
+
+    ## Last log
+    if len(dates) > 0:
+        logs[current_package] = dates
+
+    table = []
+    for k in logs:
+        first = datetime.strptime(logs[k][0], "%Y-%m-%d %H:%M:%S")
+        last = datetime.strptime(logs[k][-1], "%Y-%m-%d %H:%M:%S")
+        delta = ((last - first).seconds) / 60
+
+        table.append([str(k), str(first), str(round(delta, 2)) + "m"])
+
+    print(tabulate(table))

bob/devtools/scripts/pipelines.py

+#!/usr/bin/env python
+
+import os
+
+import click
+import gitlab
+import urllib
+
+from . import bdt
+from ..release import get_gitlab_instance
+
+from ..log import verbosity_option, get_logger, echo_normal, echo_warning
+from ..pipelines import process_log
+logger = get_logger(__name__)
+from tabulate import tabulate
+
+@click.command(
+    epilog="""
+Examples:
+
+  1. Process all the jobs from a particular pipeline
+
+     $ bdt gitlab process-pipelines bob/bob.nightlies pipelines
+
+  2. Process a particular job from a pipeline
+
+     $ bdt gitlab process-pipelines bob/bob.nightlies pipelines --job-id xxx
+
+"""
+)
+@click.argument("package")
+@click.argument("pipeline")
+@click.option('--job-id', default=None, help='A job id from a pipeline')
+@verbosity_option()
+@bdt.raise_on_error
+def process_pipelines(package, pipeline, job_id):
+    """Returns the last tag information on a given PACKAGE."""
+
+    if "/" not in package:
+        raise RuntimeError('PACKAGE should be specified as "group/name"')
+
+    gl = get_gitlab_instance()
+
+    # we lookup the gitlab package once
+    try:
+        project = gl.projects.get(package)
+        pipeline = project.pipelines.get(pipeline)
+
+        jobs = [j for j in pipeline.jobs.list()]        
+        if job_id is not None:
+            jobs = [j for j in jobs if int(j.attributes["id"])==int(job_id)]
+
+
+        if(len(jobs) == 0 ):
+            print("Job %s not found in the pipeline %s. Use `bdt gitlab get-pipelines` to search " % (job_id, pipeline.attributes["id"]))
+                         
+        # Reading log
+        try:
+            for j in jobs:
+                print("Pipeline %s, Job %s" % (pipeline.attributes["id"], int(j.attributes["id"])))
+                web_url = j.attributes["web_url"] + "/raw"                
+                log = str(urllib.request.urlopen(web_url).read()).split("\\n")
+                process_log(log)
+        except urllib.error.HTTPError as e:
+            logger.warn(
+                "Gitlab access error - Log %s can't be found" % web_url, package
+            )
+            echo_warning("%s: unknown" % (package,))
+
+        logger.info(
+            "Found gitlab project %s (id=%d)",
+            project.attributes["path_with_namespace"],
+            project.id,
+        )
+
+        pass
+    except gitlab.GitlabGetError as e:
+        logger.warn("Gitlab access error - package %s does not exist?", package)
+        echo_warning("%s: unknown" % (package,))
+
+
+@click.command(
+    epilog="""
+Examples:
+
+  1. Get the most recent pipelines from a particular project wit their corresponding job numbers
+
+     $ bdt gitlab get-pipelines bob/bob.nightlies
+
+
+"""
+)
+@click.argument("package")
+@verbosity_option()
+@bdt.raise_on_error
+def get_pipelines(package):
+    """Returns the CI pipelines given a given PACKAGE."""
+
+    if "/" not in package:
+        raise RuntimeError('PACKAGE should be specified as "group/name"')
+
+    gl = get_gitlab_instance()
+
+    # we lookup the gitlab package once
+    try:
+        project = gl.projects.get(package)
+        logger.info(
+            "Found gitlab project %s (id=%d)",
+            project.attributes["path_with_namespace"],
+            project.id,
+        )
+
+        pipelines = project.pipelines.list()
+        description = [["Pipeline", "Branch", "Status", "Jobs"]]
+        for pipeline in pipelines:
+            jobs = [j.attributes["id"] for j in pipeline.jobs.list()]
+            description.append(
+                [
+                    "%s" % pipeline.attributes["id"],
+                    "%s" % pipeline.attributes["ref"],
+                    "%s" % pipeline.attributes["status"],
+                    "%s" % jobs,
+                ]
+            )
+        print("Jobs from project %s" % package)
+        print(tabulate(description))
+
+    except gitlab.GitlabGetError as e:
+        logger.warn("Gitlab access error - package %s does not exist?", package)
+        echo_warning("%s: unknown" % (package,))

conda/meta.yaml

@@ -48,6 +48,7 @@ requirements:
     - jinja2
     - termcolor
     - psutil
+    - tabulate
 
 test:
   requires:
@@ -101,6 +102,8 @@ test:
     - bdt dav rmtree --help
     - bdt dav clean-betas --help
     - bdt dav upload --help
+    - bdt gitlab process-pipelines --help
+    - bdt gitlab get-pipelines --help    
     - sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
     - if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
 

setup.py

@@ -67,6 +67,8 @@ setup(
           'jobs = bob.devtools.scripts.jobs:jobs',
           'visibility = bob.devtools.scripts.visibility:visibility',
           'getpath = bob.devtools.scripts.getpath:getpath',
+          'process-pipelines = bob.devtools.scripts.pipelines:process_pipelines',
+          'get-pipelines- = bob.devtools.scripts.pipelines:get_pipelines',          
           ],
 
         'bdt.ci.cli': [