stats.py 8.9 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

###############################################################################
#                                                                             #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
# Contact: beat.support@idiap.ch                                              #
#                                                                             #
# This file is part of the beat.core module of the BEAT platform.             #
#                                                                             #
# Commercial License Usage                                                    #
# Licensees holding valid commercial BEAT licenses may use this file in       #
# accordance with the terms contained in a written agreement between you      #
# and Idiap. For further information contact tto@idiap.ch                     #
#                                                                             #
# Alternatively, this file may be used under the terms of the GNU Affero      #
# Public License version 3 as published by the Free Software and appearing    #
# in the file LICENSE.AGPL included in the packaging of this file.            #
# The BEAT platform is distributed in the hope that it will be useful, but    #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
#                                                                             #
# You should have received a copy of the GNU Affero Public License along      #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
#                                                                             #
###############################################################################

28
29
30
31
32
33
34
"""
=====
stats
=====

A class that can read, validate and update statistical information
"""
André Anjos's avatar
André Anjos committed
35
36
37
38
39
40
41
42
43
44
45


import os
import time
import copy

import simplejson

from . import schema
from . import prototypes

46
47
48
49
from beat.backend.python.stats import io_statistics
from beat.backend.python.stats import update


André Anjos's avatar
André Anjos committed
50
class Statistics(object):
Philip ABBET's avatar
Philip ABBET committed
51
    """Statistics define resource usage for algorithmic code runs
André Anjos's avatar
André Anjos committed
52
53


Philip ABBET's avatar
Philip ABBET committed
54
    Parameters:
André Anjos's avatar
André Anjos committed
55

André Anjos's avatar
André Anjos committed
56
      data (:py:class:`object`, Optional): The piece of data representing the
Philip ABBET's avatar
Philip ABBET committed
57
58
59
        statistics the be read, it must validate against our pre-defined
        execution schema. If the input is ``None`` or empty, then start a new
        statistics from scratch.
André Anjos's avatar
André Anjos committed
60
61


Philip ABBET's avatar
Philip ABBET committed
62
    Attributes:
André Anjos's avatar
André Anjos committed
63

Philip ABBET's avatar
Philip ABBET committed
64
65
      errors (list of str): A list containing errors found while loading this
        statistics information.
André Anjos's avatar
André Anjos committed
66

Philip ABBET's avatar
Philip ABBET committed
67
      data (dict): The original data for these statistics
André Anjos's avatar
André Anjos committed
68

Philip ABBET's avatar
Philip ABBET committed
69
    """
André Anjos's avatar
André Anjos committed
70

Philip ABBET's avatar
Philip ABBET committed
71
    def __init__(self, data=None):
André Anjos's avatar
André Anjos committed
72

Philip ABBET's avatar
Philip ABBET committed
73
        self.errors = []
André Anjos's avatar
André Anjos committed
74

Philip ABBET's avatar
Philip ABBET committed
75
76
77
78
        if data:
            self._load(data) #also runs validation
        else:
            self._data, self.errors = prototypes.load('statistics') #also validates
André Anjos's avatar
André Anjos committed
79

Philip ABBET's avatar
Philip ABBET committed
80
81
    def _load(self, data):
        """Loads the statistics
André Anjos's avatar
André Anjos committed
82

Philip ABBET's avatar
Philip ABBET committed
83
        Parameters:
André Anjos's avatar
André Anjos committed
84

Philip ABBET's avatar
Philip ABBET committed
85
86
87
88
          data (object, str, file): The piece of data to load. The input can be a
            valid python object that represents a JSON structure, a file, from
            which the JSON contents will be read out or a string. See
            :py:func:`schema.validate` for more details.
André Anjos's avatar
André Anjos committed
89

Philip ABBET's avatar
Philip ABBET committed
90
        """
André Anjos's avatar
André Anjos committed
91

Philip ABBET's avatar
Philip ABBET committed
92
93
94
        # reset
        self._data = None
        self.errors = []
André Anjos's avatar
André Anjos committed
95

Philip ABBET's avatar
Philip ABBET committed
96
97
98
99
        if not isinstance(data, dict): #user has passed a file pointer
            if not os.path.exists(data):
                self.errors.append('File not found: %s' % data)
                return
André Anjos's avatar
André Anjos committed
100

Philip ABBET's avatar
Philip ABBET committed
101
102
103
        # this runs basic validation, including JSON loading if required
        self._data, self.errors = schema.validate('statistics', data)
        if self.errors: return #don't proceed with the rest of validation
André Anjos's avatar
André Anjos committed
104
105


Philip ABBET's avatar
Philip ABBET committed
106
107
108
109
    @property
    def schema_version(self):
        """Returns the schema version"""
        return self.data.get('schema_version', 1)
André Anjos's avatar
André Anjos committed
110
111


Philip ABBET's avatar
Philip ABBET committed
112
113
114
115
    @property
    def cpu(self):
        """Returns only CPU information"""
        return self._data['cpu']
André Anjos's avatar
André Anjos committed
116

Philip ABBET's avatar
Philip ABBET committed
117
118
119
    @cpu.setter
    def cpu(self, data):
        """Sets the CPU information"""
André Anjos's avatar
André Anjos committed
120

Philip ABBET's avatar
Philip ABBET committed
121
122
        for key in ('user', 'system', 'total'):
            self._data['cpu'][key] = data[key]
André Anjos's avatar
André Anjos committed
123

Philip ABBET's avatar
Philip ABBET committed
124
125
        for key in ('voluntary', 'involuntary'):
            self._data['cpu']['context_switches'][key] = data['context_switches'][key]
André Anjos's avatar
André Anjos committed
126
127


Philip ABBET's avatar
Philip ABBET committed
128
129
130
131
    @property
    def memory(self):
        """Returns only memory information"""
        return self._data['memory']
André Anjos's avatar
André Anjos committed
132
133


Philip ABBET's avatar
Philip ABBET committed
134
135
136
    @memory.setter
    def memory(self, data):
        """Sets only the memory information"""
André Anjos's avatar
André Anjos committed
137

Philip ABBET's avatar
Philip ABBET committed
138
        for key in ('rss',): self._data['memory'][key] = data[key]
André Anjos's avatar
André Anjos committed
139
140


Philip ABBET's avatar
Philip ABBET committed
141
142
143
144
    @property
    def data(self):
        """Returns only I/O information"""
        return self._data['data']
André Anjos's avatar
André Anjos committed
145
146


Philip ABBET's avatar
Philip ABBET committed
147
148
149
    @data.setter
    def data(self, data):
        """Sets only the I/O information"""
André Anjos's avatar
André Anjos committed
150

Philip ABBET's avatar
Philip ABBET committed
151
152
153
        for key in ('volume', 'blocks', 'time'):
            self._data['data'][key]['read'] = data[key]['read']
            self._data['data'][key]['write'] = data[key]['write']
André Anjos's avatar
André Anjos committed
154

Philip ABBET's avatar
Philip ABBET committed
155
156
        self._data['data']['files'] = list(data['files'])
        self._data['network'] = data['network']
André Anjos's avatar
André Anjos committed
157
158


Philip ABBET's avatar
Philip ABBET committed
159
160
161
    @property
    def valid(self):
        """A boolean that indicates if this executor is valid or not"""
André Anjos's avatar
André Anjos committed
162

Philip ABBET's avatar
Philip ABBET committed
163
        return not bool(self.errors)
André Anjos's avatar
André Anjos committed
164
165


Philip ABBET's avatar
Philip ABBET committed
166
167
    def __add__(self, other):
        """Adds two statistics data blocks"""
André Anjos's avatar
André Anjos committed
168

Philip ABBET's avatar
Philip ABBET committed
169
170
171
        retval = Statistics(copy.deepcopy(self._data))
        retval += other
        return retval
André Anjos's avatar
André Anjos committed
172
173


Philip ABBET's avatar
Philip ABBET committed
174
175
    def __iadd__(self, other):
        """Self-add statistics from another block"""
André Anjos's avatar
André Anjos committed
176

Philip ABBET's avatar
Philip ABBET committed
177
        if not isinstance(other, Statistics): return NotImplemented
André Anjos's avatar
André Anjos committed
178

Philip ABBET's avatar
Philip ABBET committed
179
180
        for key in ('user', 'system', 'total'):
            self._data['cpu'][key] += other._data['cpu'][key]
André Anjos's avatar
André Anjos committed
181

Philip ABBET's avatar
Philip ABBET committed
182
183
184
        for key in ('voluntary', 'involuntary'):
            self._data['cpu']['context_switches'][key] += \
                    other._data['cpu']['context_switches'][key]
André Anjos's avatar
André Anjos committed
185

Philip ABBET's avatar
Philip ABBET committed
186
187
188
        for key in ('rss', ): #gets the maximum between the two
            self._data['memory'][key] = max(other._data['memory'][key],
                self._data['memory'][key])
André Anjos's avatar
André Anjos committed
189

Philip ABBET's avatar
Philip ABBET committed
190
191
192
        for key in ('volume', 'blocks', 'time'):
            self._data['data'][key]['read'] += other._data['data'][key]['read']
            self._data['data'][key]['write'] += other._data['data'][key]['write']
André Anjos's avatar
André Anjos committed
193

Philip ABBET's avatar
Philip ABBET committed
194
        self._data['data']['files'] += other._data['data']['files']
André Anjos's avatar
André Anjos committed
195

Philip ABBET's avatar
Philip ABBET committed
196
197
        self._data['data']['network']['wait_time'] += \
                other._data['data']['network']['wait_time']
André Anjos's avatar
André Anjos committed
198

Philip ABBET's avatar
Philip ABBET committed
199
        return self
André Anjos's avatar
André Anjos committed
200
201


Philip ABBET's avatar
Philip ABBET committed
202
    def __str__(self):
André Anjos's avatar
André Anjos committed
203

Philip ABBET's avatar
Philip ABBET committed
204
        return self.as_json(2)
André Anjos's avatar
André Anjos committed
205
206


Philip ABBET's avatar
Philip ABBET committed
207
    def as_json(self, indent=None):
André Anjos's avatar
André Anjos committed
208

Philip ABBET's avatar
Philip ABBET committed
209
        return simplejson.dumps(self._data, indent=indent)
André Anjos's avatar
André Anjos committed
210
211


Philip ABBET's avatar
Philip ABBET committed
212
    def as_dict(self):
André Anjos's avatar
André Anjos committed
213

Philip ABBET's avatar
Philip ABBET committed
214
        return self._data
André Anjos's avatar
André Anjos committed
215
216


Philip ABBET's avatar
Philip ABBET committed
217
218
    def write(self, f):
        """Writes contents to a file-like object"""
André Anjos's avatar
André Anjos committed
219

Philip ABBET's avatar
Philip ABBET committed
220
221
222
        if hasattr(f, 'write'): f.write(str(self))
        else:
            with open(f, 'wt') as fobj: fobj.write(str(self))
André Anjos's avatar
André Anjos committed
223
224


225
#----------------------------------------------------------
André Anjos's avatar
André Anjos committed
226
227


228
def cpu_statistics(start, end):
Philip ABBET's avatar
Philip ABBET committed
229
    """Summarizes current CPU usage
André Anjos's avatar
André Anjos committed
230

Philip ABBET's avatar
Philip ABBET committed
231
232
233
234
    This method should be used when the currently set algorithm is the only one
    executed through the whole process. It is done for collecting resource
    statistics on separate processing environments. It follows the recipe in:
    http://stackoverflow.com/questions/30271942/get-docker-container-cpu-usage-as-percentage
André Anjos's avatar
André Anjos committed
235

Philip ABBET's avatar
Philip ABBET committed
236
    Returns:
André Anjos's avatar
André Anjos committed
237

Philip ABBET's avatar
Philip ABBET committed
238
      dict: A dictionary summarizing current CPU usage
André Anjos's avatar
André Anjos committed
239

Philip ABBET's avatar
Philip ABBET committed
240
    """
André Anjos's avatar
André Anjos committed
241

242
    if 'system_cpu_usage' not in end:
Philip ABBET's avatar
Philip ABBET committed
243
244
245
246
247
248
249
        return {
                'user': 0.0,
                'system': 0.0,
                'total': 0.0,
                'percent': 0.0,
                'processors': 1,
               }
Philip ABBET's avatar
Philip ABBET committed
250

Philip ABBET's avatar
Philip ABBET committed
251
252
253
254
    if start is not None:
        user_cpu = end['cpu_usage']['total_usage'] - \
            start['cpu_usage']['total_usage']
        total_cpu = end['system_cpu_usage'] - start['system_cpu_usage']
255

Philip ABBET's avatar
Philip ABBET committed
256
257
258
    else:
        user_cpu = end['cpu_usage']['total_usage']
        total_cpu = end['system_cpu_usage']
259

Philip ABBET's avatar
Philip ABBET committed
260
261
262
263
    user_cpu /= 1000000000. #in seconds
    total_cpu /= 1000000000. #in seconds
    processors = len(end['cpu_usage']['percpu_usage']) if \
        end['cpu_usage']['percpu_usage'] is not None else 1
264

Philip ABBET's avatar
Philip ABBET committed
265
266
267
268
269
270
271
    return {
            'user': user_cpu,
            'system': 0.,
            'total': total_cpu,
            'percent': 100.*processors*user_cpu/total_cpu if total_cpu else 0.,
            'processors': processors,
           }
André Anjos's avatar
André Anjos committed
272
273


274
275
276
#----------------------------------------------------------


277
def memory_statistics(data):
Philip ABBET's avatar
Philip ABBET committed
278
    """Summarizes current memory usage
André Anjos's avatar
André Anjos committed
279

Philip ABBET's avatar
Philip ABBET committed
280
281
282
    This method should be used when the currently set algorithm is the only one
    executed through the whole process. It is done for collecting resource
    statistics on separate processing environments.
André Anjos's avatar
André Anjos committed
283

Philip ABBET's avatar
Philip ABBET committed
284
    Returns:
André Anjos's avatar
André Anjos committed
285

Philip ABBET's avatar
Philip ABBET committed
286
      dict: A dictionary summarizing current memory usage
André Anjos's avatar
André Anjos committed
287

Philip ABBET's avatar
Philip ABBET committed
288
    """
André Anjos's avatar
André Anjos committed
289

Philip ABBET's avatar
Philip ABBET committed
290
291
    limit = float(data['limit'])
    memory = float(data['max_usage'])
André Anjos's avatar
André Anjos committed
292

Philip ABBET's avatar
Philip ABBET committed
293
294
295
296
297
    return {
            'rss': memory,
            'limit': limit,
            'percent': 100.*memory/limit if limit else 0.,
           }