stats.py 9.29 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

###############################################################################
#                                                                             #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
# Contact: beat.support@idiap.ch                                              #
#                                                                             #
# This file is part of the beat.core module of the BEAT platform.             #
#                                                                             #
# Commercial License Usage                                                    #
# Licensees holding valid commercial BEAT licenses may use this file in       #
# accordance with the terms contained in a written agreement between you      #
# and Idiap. For further information contact tto@idiap.ch                     #
#                                                                             #
# Alternatively, this file may be used under the terms of the GNU Affero      #
# Public License version 3 as published by the Free Software and appearing    #
# in the file LICENSE.AGPL included in the packaging of this file.            #
# The BEAT platform is distributed in the hope that it will be useful, but    #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
#                                                                             #
# You should have received a copy of the GNU Affero Public License along      #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
#                                                                             #
###############################################################################

28
29
30
31
32
33
"""
=====
stats
=====

A class that can read, validate and update statistical information
34
35
36
37

Forward impored from :py:mod:`beat.backend.python.stats`:
:py:func:`beat.backend.python.stats.io_statistics`
:py:func:`beat.backend.python.stats.update`
38
"""
André Anjos's avatar
André Anjos committed
39
40
41
42
43
44
45
46
47
48


import os
import copy

import simplejson

from . import schema
from . import prototypes

49
50
51
52
from beat.backend.python.stats import io_statistics
from beat.backend.python.stats import update


André Anjos's avatar
André Anjos committed
53
class Statistics(object):
Philip ABBET's avatar
Philip ABBET committed
54
    """Statistics define resource usage for algorithmic code runs
André Anjos's avatar
André Anjos committed
55
56


Philip ABBET's avatar
Philip ABBET committed
57
    Parameters:
André Anjos's avatar
André Anjos committed
58

André Anjos's avatar
André Anjos committed
59
      data (:py:class:`object`, Optional): The piece of data representing the
Philip ABBET's avatar
Philip ABBET committed
60
61
62
        statistics the be read, it must validate against our pre-defined
        execution schema. If the input is ``None`` or empty, then start a new
        statistics from scratch.
André Anjos's avatar
André Anjos committed
63
64


Philip ABBET's avatar
Philip ABBET committed
65
    Attributes:
André Anjos's avatar
André Anjos committed
66

67
      errors (list): A list strings containing errors found while loading this
Philip ABBET's avatar
Philip ABBET committed
68
        statistics information.
André Anjos's avatar
André Anjos committed
69

Philip ABBET's avatar
Philip ABBET committed
70
      data (dict): The original data for these statistics
André Anjos's avatar
André Anjos committed
71

Philip ABBET's avatar
Philip ABBET committed
72
    """
André Anjos's avatar
André Anjos committed
73

Philip ABBET's avatar
Philip ABBET committed
74
    def __init__(self, data=None):
André Anjos's avatar
André Anjos committed
75

Philip ABBET's avatar
Philip ABBET committed
76
        self.errors = []
André Anjos's avatar
André Anjos committed
77

Philip ABBET's avatar
Philip ABBET committed
78
        if data:
79
            self._load(data)  # also runs validation
Philip ABBET's avatar
Philip ABBET committed
80
        else:
81
            self._data, self.errors = prototypes.load('statistics')  # also validates
André Anjos's avatar
André Anjos committed
82

Philip ABBET's avatar
Philip ABBET committed
83
84
    def _load(self, data):
        """Loads the statistics
André Anjos's avatar
André Anjos committed
85

Philip ABBET's avatar
Philip ABBET committed
86
        Parameters:
André Anjos's avatar
André Anjos committed
87

88
89
90
          data (object, str, file): The piece of data to load. The input can be
            a valid python object that represents a JSON structure, a file,
            from which the JSON contents will be read out or a string. See
Philip ABBET's avatar
Philip ABBET committed
91
92
            :py:func:`schema.validate` for more details.
        """
André Anjos's avatar
André Anjos committed
93

Philip ABBET's avatar
Philip ABBET committed
94
95
96
        # reset
        self._data = None
        self.errors = []
André Anjos's avatar
André Anjos committed
97

98
        if not isinstance(data, dict):  # user has passed a file pointer
Philip ABBET's avatar
Philip ABBET committed
99
100
101
            if not os.path.exists(data):
                self.errors.append('File not found: %s' % data)
                return
André Anjos's avatar
André Anjos committed
102

Philip ABBET's avatar
Philip ABBET committed
103
104
        # this runs basic validation, including JSON loading if required
        self._data, self.errors = schema.validate('statistics', data)
105
        if self.errors: return  # don't proceed with the rest of validation
André Anjos's avatar
André Anjos committed
106
107


Philip ABBET's avatar
Philip ABBET committed
108
109
110
    @property
    def schema_version(self):
        """Returns the schema version"""
111

Philip ABBET's avatar
Philip ABBET committed
112
        return self.data.get('schema_version', 1)
André Anjos's avatar
André Anjos committed
113
114


Philip ABBET's avatar
Philip ABBET committed
115
116
117
    @property
    def cpu(self):
        """Returns only CPU information"""
118

Philip ABBET's avatar
Philip ABBET committed
119
        return self._data['cpu']
André Anjos's avatar
André Anjos committed
120

Philip ABBET's avatar
Philip ABBET committed
121
122
123
    @cpu.setter
    def cpu(self, data):
        """Sets the CPU information"""
André Anjos's avatar
André Anjos committed
124

Philip ABBET's avatar
Philip ABBET committed
125
126
        for key in ('user', 'system', 'total'):
            self._data['cpu'][key] = data[key]
André Anjos's avatar
André Anjos committed
127

Philip ABBET's avatar
Philip ABBET committed
128
129
        for key in ('voluntary', 'involuntary'):
            self._data['cpu']['context_switches'][key] = data['context_switches'][key]
André Anjos's avatar
André Anjos committed
130
131


Philip ABBET's avatar
Philip ABBET committed
132
133
134
    @property
    def memory(self):
        """Returns only memory information"""
135

Philip ABBET's avatar
Philip ABBET committed
136
        return self._data['memory']
André Anjos's avatar
André Anjos committed
137
138


Philip ABBET's avatar
Philip ABBET committed
139
140
141
    @memory.setter
    def memory(self, data):
        """Sets only the memory information"""
André Anjos's avatar
André Anjos committed
142

Philip ABBET's avatar
Philip ABBET committed
143
        for key in ('rss',): self._data['memory'][key] = data[key]
André Anjos's avatar
André Anjos committed
144
145


Philip ABBET's avatar
Philip ABBET committed
146
147
148
    @property
    def data(self):
        """Returns only I/O information"""
149

Philip ABBET's avatar
Philip ABBET committed
150
        return self._data['data']
André Anjos's avatar
André Anjos committed
151
152


Philip ABBET's avatar
Philip ABBET committed
153
154
155
    @data.setter
    def data(self, data):
        """Sets only the I/O information"""
André Anjos's avatar
André Anjos committed
156

Philip ABBET's avatar
Philip ABBET committed
157
158
159
        for key in ('volume', 'blocks', 'time'):
            self._data['data'][key]['read'] = data[key]['read']
            self._data['data'][key]['write'] = data[key]['write']
André Anjos's avatar
André Anjos committed
160

Philip ABBET's avatar
Philip ABBET committed
161
162
        self._data['data']['files'] = list(data['files'])
        self._data['network'] = data['network']
André Anjos's avatar
André Anjos committed
163
164


Philip ABBET's avatar
Philip ABBET committed
165
166
167
    @property
    def valid(self):
        """A boolean that indicates if this executor is valid or not"""
André Anjos's avatar
André Anjos committed
168

Philip ABBET's avatar
Philip ABBET committed
169
        return not bool(self.errors)
André Anjos's avatar
André Anjos committed
170
171


Philip ABBET's avatar
Philip ABBET committed
172
173
    def __add__(self, other):
        """Adds two statistics data blocks"""
André Anjos's avatar
André Anjos committed
174

Philip ABBET's avatar
Philip ABBET committed
175
176
177
        retval = Statistics(copy.deepcopy(self._data))
        retval += other
        return retval
André Anjos's avatar
André Anjos committed
178
179


Philip ABBET's avatar
Philip ABBET committed
180
181
    def __iadd__(self, other):
        """Self-add statistics from another block"""
André Anjos's avatar
André Anjos committed
182

Philip ABBET's avatar
Philip ABBET committed
183
        if not isinstance(other, Statistics): return NotImplemented
André Anjos's avatar
André Anjos committed
184

Philip ABBET's avatar
Philip ABBET committed
185
186
        for key in ('user', 'system', 'total'):
            self._data['cpu'][key] += other._data['cpu'][key]
André Anjos's avatar
André Anjos committed
187

Philip ABBET's avatar
Philip ABBET committed
188
189
190
        for key in ('voluntary', 'involuntary'):
            self._data['cpu']['context_switches'][key] += \
                    other._data['cpu']['context_switches'][key]
André Anjos's avatar
André Anjos committed
191

Philip ABBET's avatar
Philip ABBET committed
192
193
194
        for key in ('rss', ): #gets the maximum between the two
            self._data['memory'][key] = max(other._data['memory'][key],
                self._data['memory'][key])
André Anjos's avatar
André Anjos committed
195

Philip ABBET's avatar
Philip ABBET committed
196
197
198
        for key in ('volume', 'blocks', 'time'):
            self._data['data'][key]['read'] += other._data['data'][key]['read']
            self._data['data'][key]['write'] += other._data['data'][key]['write']
André Anjos's avatar
André Anjos committed
199

Philip ABBET's avatar
Philip ABBET committed
200
        self._data['data']['files'] += other._data['data']['files']
André Anjos's avatar
André Anjos committed
201

Philip ABBET's avatar
Philip ABBET committed
202
203
        self._data['data']['network']['wait_time'] += \
                other._data['data']['network']['wait_time']
André Anjos's avatar
André Anjos committed
204

Philip ABBET's avatar
Philip ABBET committed
205
        return self
André Anjos's avatar
André Anjos committed
206
207


Philip ABBET's avatar
Philip ABBET committed
208
    def __str__(self):
André Anjos's avatar
André Anjos committed
209

Philip ABBET's avatar
Philip ABBET committed
210
        return self.as_json(2)
André Anjos's avatar
André Anjos committed
211
212


Philip ABBET's avatar
Philip ABBET committed
213
    def as_json(self, indent=None):
214
215
216
217
218
219
220
221
        """Returns self as as JSON

        Parameters:
            :param indent int: Indentation to use for the JSON generation

        Returns:
            dict: JSON representation
        """
André Anjos's avatar
André Anjos committed
222

Philip ABBET's avatar
Philip ABBET committed
223
        return simplejson.dumps(self._data, indent=indent)
André Anjos's avatar
André Anjos committed
224
225


Philip ABBET's avatar
Philip ABBET committed
226
    def as_dict(self):
227
        """Returns self as a dictionary"""
André Anjos's avatar
André Anjos committed
228

Philip ABBET's avatar
Philip ABBET committed
229
        return self._data
André Anjos's avatar
André Anjos committed
230
231


Philip ABBET's avatar
Philip ABBET committed
232
233
    def write(self, f):
        """Writes contents to a file-like object"""
André Anjos's avatar
André Anjos committed
234

Philip ABBET's avatar
Philip ABBET committed
235
236
237
        if hasattr(f, 'write'): f.write(str(self))
        else:
            with open(f, 'wt') as fobj: fobj.write(str(self))
André Anjos's avatar
André Anjos committed
238
239


240
# ----------------------------------------------------------
André Anjos's avatar
André Anjos committed
241
242


243
def cpu_statistics(start, end):
Philip ABBET's avatar
Philip ABBET committed
244
    """Summarizes current CPU usage
André Anjos's avatar
André Anjos committed
245

Philip ABBET's avatar
Philip ABBET committed
246
247
248
249
    This method should be used when the currently set algorithm is the only one
    executed through the whole process. It is done for collecting resource
    statistics on separate processing environments. It follows the recipe in:
    http://stackoverflow.com/questions/30271942/get-docker-container-cpu-usage-as-percentage
André Anjos's avatar
André Anjos committed
250

Philip ABBET's avatar
Philip ABBET committed
251
    Returns:
André Anjos's avatar
André Anjos committed
252

Philip ABBET's avatar
Philip ABBET committed
253
      dict: A dictionary summarizing current CPU usage
André Anjos's avatar
André Anjos committed
254

Philip ABBET's avatar
Philip ABBET committed
255
    """
André Anjos's avatar
André Anjos committed
256

257
    if 'system_cpu_usage' not in end:
Philip ABBET's avatar
Philip ABBET committed
258
259
260
261
262
263
264
        return {
                'user': 0.0,
                'system': 0.0,
                'total': 0.0,
                'percent': 0.0,
                'processors': 1,
               }
Philip ABBET's avatar
Philip ABBET committed
265

Philip ABBET's avatar
Philip ABBET committed
266
267
268
269
    if start is not None:
        user_cpu = end['cpu_usage']['total_usage'] - \
            start['cpu_usage']['total_usage']
        total_cpu = end['system_cpu_usage'] - start['system_cpu_usage']
270

Philip ABBET's avatar
Philip ABBET committed
271
272
273
    else:
        user_cpu = end['cpu_usage']['total_usage']
        total_cpu = end['system_cpu_usage']
274

275
276
    user_cpu /= 1000000000.  # in seconds
    total_cpu /= 1000000000.  # in seconds
Philip ABBET's avatar
Philip ABBET committed
277
278
    processors = len(end['cpu_usage']['percpu_usage']) if \
        end['cpu_usage']['percpu_usage'] is not None else 1
279

Philip ABBET's avatar
Philip ABBET committed
280
281
282
283
284
285
286
    return {
            'user': user_cpu,
            'system': 0.,
            'total': total_cpu,
            'percent': 100.*processors*user_cpu/total_cpu if total_cpu else 0.,
            'processors': processors,
           }
André Anjos's avatar
André Anjos committed
287
288


289
# ----------------------------------------------------------
290
291


292
def memory_statistics(data):
Philip ABBET's avatar
Philip ABBET committed
293
    """Summarizes current memory usage
André Anjos's avatar
André Anjos committed
294

Philip ABBET's avatar
Philip ABBET committed
295
296
297
    This method should be used when the currently set algorithm is the only one
    executed through the whole process. It is done for collecting resource
    statistics on separate processing environments.
André Anjos's avatar
André Anjos committed
298

Philip ABBET's avatar
Philip ABBET committed
299
    Returns:
André Anjos's avatar
André Anjos committed
300

Philip ABBET's avatar
Philip ABBET committed
301
      dict: A dictionary summarizing current memory usage
André Anjos's avatar
André Anjos committed
302

Philip ABBET's avatar
Philip ABBET committed
303
    """
André Anjos's avatar
André Anjos committed
304

Philip ABBET's avatar
Philip ABBET committed
305
306
    limit = float(data['limit'])
    memory = float(data['max_usage'])
André Anjos's avatar
André Anjos committed
307

Philip ABBET's avatar
Philip ABBET committed
308
309
310
311
312
    return {
            'rss': memory,
            'limit': limit,
            'percent': 100.*memory/limit if limit else 0.,
           }