Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
bob
bob.devtools
Commits
5d11ace6
Commit
5d11ace6
authored
Oct 21, 2019
by
Tiago de Freitas Pereira
Browse files
Created 2 commands to process gitlab logs
parent
44852cc2
Pipeline
#34590
passed with stage
in 5 minutes and 37 seconds
Changes
4
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
bob/devtools/pipelines.py
0 → 100644
View file @
5d11ace6
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Pipeline utilities"""
from
tabulate
import
tabulate
import
re
from
datetime
import
datetime
def
process_log
(
log
):
"""
Summarizes the execution time of a pipeline given its Job log
"""
current_package
=
None
logs
=
dict
()
dates
=
[]
for
l
in
log
:
# Check which package are we
if
len
(
re
.
findall
(
"Building bob/[a-z]*"
,
l
))
>
0
:
logs
[
current_package
]
=
dates
dates
=
[]
pattern
=
re
.
findall
(
"Building bob/[a-z]*"
,
l
)[
0
]
current_package
=
l
[
9
:
-
1
]
continue
# Checking the date
date
=
re
.
findall
(
"[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2} [0-9]{2,2}:[0-9]{2,2}:[0-9]{2,2}"
,
l
)
if
len
(
date
)
>
0
:
# logs[date[0]]=current_package
dates
.
append
(
date
[
0
])
## Last log
if
len
(
dates
)
>
0
:
logs
[
current_package
]
=
dates
table
=
[]
for
k
in
logs
:
first
=
datetime
.
strptime
(
logs
[
k
][
0
],
"%Y-%m-%d %H:%M:%S"
)
last
=
datetime
.
strptime
(
logs
[
k
][
-
1
],
"%Y-%m-%d %H:%M:%S"
)
delta
=
((
last
-
first
).
seconds
)
/
60
table
.
append
([
str
(
k
),
str
(
first
),
str
(
round
(
delta
,
2
))
+
"m"
])
print
(
tabulate
(
table
))
bob/devtools/scripts/pipelines.py
0 → 100644
View file @
5d11ace6
#!/usr/bin/env python
import
os
import
click
import
gitlab
import
urllib
from
.
import
bdt
from
..release
import
get_gitlab_instance
from
..log
import
verbosity_option
,
get_logger
,
echo_normal
,
echo_warning
from
..pipelines
import
process_log
logger
=
get_logger
(
__name__
)
from
tabulate
import
tabulate
@
click
.
command
(
epilog
=
"""
Examples:
1. Process all the jobs from a particular pipeline
$ bdt gitlab process-pipelines bob/bob.nightlies pipelines
2. Process a particular job from a pipeline
$ bdt gitlab process-pipelines bob/bob.nightlies pipelines --job-id xxx
"""
)
@
click
.
argument
(
"package"
)
@
click
.
argument
(
"pipeline"
)
@
click
.
option
(
'--job-id'
,
default
=
None
,
help
=
'A job id from a pipeline'
)
@
verbosity_option
()
@
bdt
.
raise_on_error
def
process_pipelines
(
package
,
pipeline
,
job_id
):
"""Returns the last tag information on a given PACKAGE."""
if
"/"
not
in
package
:
raise
RuntimeError
(
'PACKAGE should be specified as "group/name"'
)
gl
=
get_gitlab_instance
()
# we lookup the gitlab package once
try
:
project
=
gl
.
projects
.
get
(
package
)
pipeline
=
project
.
pipelines
.
get
(
pipeline
)
jobs
=
[
j
for
j
in
pipeline
.
jobs
.
list
()]
if
job_id
is
not
None
:
jobs
=
[
j
for
j
in
jobs
if
int
(
j
.
attributes
[
"id"
])
==
int
(
job_id
)]
if
(
len
(
jobs
)
==
0
):
print
(
"Job %s not found in the pipeline %s. Use `bdt gitlab get-pipelines` to search "
%
(
job_id
,
pipeline
.
attributes
[
"id"
]))
# Reading log
try
:
for
j
in
jobs
:
print
(
"Pipeline %s, Job %s"
%
(
pipeline
.
attributes
[
"id"
],
int
(
j
.
attributes
[
"id"
])))
web_url
=
j
.
attributes
[
"web_url"
]
+
"/raw"
log
=
str
(
urllib
.
request
.
urlopen
(
web_url
).
read
()).
split
(
"
\\
n"
)
process_log
(
log
)
except
urllib
.
error
.
HTTPError
as
e
:
logger
.
warn
(
"Gitlab access error - Log %s can't be found"
%
web_url
,
package
)
echo_warning
(
"%s: unknown"
%
(
package
,))
logger
.
info
(
"Found gitlab project %s (id=%d)"
,
project
.
attributes
[
"path_with_namespace"
],
project
.
id
,
)
pass
except
gitlab
.
GitlabGetError
as
e
:
logger
.
warn
(
"Gitlab access error - package %s does not exist?"
,
package
)
echo_warning
(
"%s: unknown"
%
(
package
,))
@
click
.
command
(
epilog
=
"""
Examples:
1. Get the most recent pipelines from a particular project wit their corresponding job numbers
$ bdt gitlab get-pipelines bob/bob.nightlies
"""
)
@
click
.
argument
(
"package"
)
@
verbosity_option
()
@
bdt
.
raise_on_error
def
get_pipelines
(
package
):
"""Returns the CI pipelines given a given PACKAGE."""
if
"/"
not
in
package
:
raise
RuntimeError
(
'PACKAGE should be specified as "group/name"'
)
gl
=
get_gitlab_instance
()
# we lookup the gitlab package once
try
:
project
=
gl
.
projects
.
get
(
package
)
logger
.
info
(
"Found gitlab project %s (id=%d)"
,
project
.
attributes
[
"path_with_namespace"
],
project
.
id
,
)
pipelines
=
project
.
pipelines
.
list
()
description
=
[[
"Pipeline"
,
"Branch"
,
"Status"
,
"Jobs"
]]
for
pipeline
in
pipelines
:
jobs
=
[
j
.
attributes
[
"id"
]
for
j
in
pipeline
.
jobs
.
list
()]
description
.
append
(
[
"%s"
%
pipeline
.
attributes
[
"id"
],
"%s"
%
pipeline
.
attributes
[
"ref"
],
"%s"
%
pipeline
.
attributes
[
"status"
],
"%s"
%
jobs
,
]
)
print
(
"Jobs from project %s"
%
package
)
print
(
tabulate
(
description
))
except
gitlab
.
GitlabGetError
as
e
:
logger
.
warn
(
"Gitlab access error - package %s does not exist?"
,
package
)
echo_warning
(
"%s: unknown"
%
(
package
,))
conda/meta.yaml
View file @
5d11ace6
...
...
@@ -48,6 +48,7 @@ requirements:
-
jinja2
-
termcolor
-
psutil
-
tabulate
test
:
requires
:
...
...
@@ -101,6 +102,8 @@ test:
-
bdt dav rmtree --help
-
bdt dav clean-betas --help
-
bdt dav upload --help
-
bdt gitlab process-pipelines --help
-
bdt gitlab get-pipelines --help
-
sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
-
if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
...
...
setup.py
View file @
5d11ace6
...
...
@@ -67,6 +67,8 @@ setup(
'jobs = bob.devtools.scripts.jobs:jobs'
,
'visibility = bob.devtools.scripts.visibility:visibility'
,
'getpath = bob.devtools.scripts.getpath:getpath'
,
'process-pipelines = bob.devtools.scripts.pipelines:process_pipelines'
,
'get-pipelines- = bob.devtools.scripts.pipelines:get_pipelines'
,
],
'bdt.ci.cli'
:
[
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment