stats.py 8.87 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

###############################################################################
#                                                                             #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
# Contact: beat.support@idiap.ch                                              #
#                                                                             #
# This file is part of the beat.core module of the BEAT platform.             #
#                                                                             #
# Commercial License Usage                                                    #
# Licensees holding valid commercial BEAT licenses may use this file in       #
# accordance with the terms contained in a written agreement between you      #
# and Idiap. For further information contact tto@idiap.ch                     #
#                                                                             #
# Alternatively, this file may be used under the terms of the GNU Affero      #
# Public License version 3 as published by the Free Software and appearing    #
# in the file LICENSE.AGPL included in the packaging of this file.            #
# The BEAT platform is distributed in the hope that it will be useful, but    #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
#                                                                             #
# You should have received a copy of the GNU Affero Public License along      #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
#                                                                             #
###############################################################################


'''A class that can read, validate and update statistical information'''

import os
import time
import copy

import simplejson

from . import schema
from . import prototypes

40
41
42
43
from beat.backend.python.stats import io_statistics
from beat.backend.python.stats import update


André Anjos's avatar
André Anjos committed
44
class Statistics(object):
Philip ABBET's avatar
Philip ABBET committed
45
    """Statistics define resource usage for algorithmic code runs
André Anjos's avatar
André Anjos committed
46
47


Philip ABBET's avatar
Philip ABBET committed
48
    Parameters:
André Anjos's avatar
André Anjos committed
49

Philip ABBET's avatar
Philip ABBET committed
50
51
52
53
      data (object, optional): The piece of data representing the
        statistics the be read, it must validate against our pre-defined
        execution schema. If the input is ``None`` or empty, then start a new
        statistics from scratch.
André Anjos's avatar
André Anjos committed
54
55


Philip ABBET's avatar
Philip ABBET committed
56
    Attributes:
André Anjos's avatar
André Anjos committed
57

Philip ABBET's avatar
Philip ABBET committed
58
59
      errors (list of str): A list containing errors found while loading this
        statistics information.
André Anjos's avatar
André Anjos committed
60

Philip ABBET's avatar
Philip ABBET committed
61
      data (dict): The original data for these statistics
André Anjos's avatar
André Anjos committed
62

Philip ABBET's avatar
Philip ABBET committed
63
    """
André Anjos's avatar
André Anjos committed
64

Philip ABBET's avatar
Philip ABBET committed
65
    def __init__(self, data=None):
André Anjos's avatar
André Anjos committed
66

Philip ABBET's avatar
Philip ABBET committed
67
        self.errors = []
André Anjos's avatar
André Anjos committed
68

Philip ABBET's avatar
Philip ABBET committed
69
70
71
72
        if data:
            self._load(data) #also runs validation
        else:
            self._data, self.errors = prototypes.load('statistics') #also validates
André Anjos's avatar
André Anjos committed
73

Philip ABBET's avatar
Philip ABBET committed
74
75
    def _load(self, data):
        """Loads the statistics
André Anjos's avatar
André Anjos committed
76

Philip ABBET's avatar
Philip ABBET committed
77
        Parameters:
André Anjos's avatar
André Anjos committed
78

Philip ABBET's avatar
Philip ABBET committed
79
80
81
82
          data (object, str, file): The piece of data to load. The input can be a
            valid python object that represents a JSON structure, a file, from
            which the JSON contents will be read out or a string. See
            :py:func:`schema.validate` for more details.
André Anjos's avatar
André Anjos committed
83

Philip ABBET's avatar
Philip ABBET committed
84
        """
André Anjos's avatar
André Anjos committed
85

Philip ABBET's avatar
Philip ABBET committed
86
87
88
        # reset
        self._data = None
        self.errors = []
André Anjos's avatar
André Anjos committed
89

Philip ABBET's avatar
Philip ABBET committed
90
91
92
93
        if not isinstance(data, dict): #user has passed a file pointer
            if not os.path.exists(data):
                self.errors.append('File not found: %s' % data)
                return
André Anjos's avatar
André Anjos committed
94

Philip ABBET's avatar
Philip ABBET committed
95
96
97
        # this runs basic validation, including JSON loading if required
        self._data, self.errors = schema.validate('statistics', data)
        if self.errors: return #don't proceed with the rest of validation
André Anjos's avatar
André Anjos committed
98
99


Philip ABBET's avatar
Philip ABBET committed
100
101
102
103
    @property
    def schema_version(self):
        """Returns the schema version"""
        return self.data.get('schema_version', 1)
André Anjos's avatar
André Anjos committed
104
105


Philip ABBET's avatar
Philip ABBET committed
106
107
108
109
    @property
    def cpu(self):
        """Returns only CPU information"""
        return self._data['cpu']
André Anjos's avatar
André Anjos committed
110

Philip ABBET's avatar
Philip ABBET committed
111
112
113
    @cpu.setter
    def cpu(self, data):
        """Sets the CPU information"""
André Anjos's avatar
André Anjos committed
114

Philip ABBET's avatar
Philip ABBET committed
115
116
        for key in ('user', 'system', 'total'):
            self._data['cpu'][key] = data[key]
André Anjos's avatar
André Anjos committed
117

Philip ABBET's avatar
Philip ABBET committed
118
119
        for key in ('voluntary', 'involuntary'):
            self._data['cpu']['context_switches'][key] = data['context_switches'][key]
André Anjos's avatar
André Anjos committed
120
121


Philip ABBET's avatar
Philip ABBET committed
122
123
124
125
    @property
    def memory(self):
        """Returns only memory information"""
        return self._data['memory']
André Anjos's avatar
André Anjos committed
126
127


Philip ABBET's avatar
Philip ABBET committed
128
129
130
    @memory.setter
    def memory(self, data):
        """Sets only the memory information"""
André Anjos's avatar
André Anjos committed
131

Philip ABBET's avatar
Philip ABBET committed
132
        for key in ('rss',): self._data['memory'][key] = data[key]
André Anjos's avatar
André Anjos committed
133
134


Philip ABBET's avatar
Philip ABBET committed
135
136
137
138
    @property
    def data(self):
        """Returns only I/O information"""
        return self._data['data']
André Anjos's avatar
André Anjos committed
139
140


Philip ABBET's avatar
Philip ABBET committed
141
142
143
    @data.setter
    def data(self, data):
        """Sets only the I/O information"""
André Anjos's avatar
André Anjos committed
144

Philip ABBET's avatar
Philip ABBET committed
145
146
147
        for key in ('volume', 'blocks', 'time'):
            self._data['data'][key]['read'] = data[key]['read']
            self._data['data'][key]['write'] = data[key]['write']
André Anjos's avatar
André Anjos committed
148

Philip ABBET's avatar
Philip ABBET committed
149
150
        self._data['data']['files'] = list(data['files'])
        self._data['network'] = data['network']
André Anjos's avatar
André Anjos committed
151
152


Philip ABBET's avatar
Philip ABBET committed
153
154
155
    @property
    def valid(self):
        """A boolean that indicates if this executor is valid or not"""
André Anjos's avatar
André Anjos committed
156

Philip ABBET's avatar
Philip ABBET committed
157
        return not bool(self.errors)
André Anjos's avatar
André Anjos committed
158
159


Philip ABBET's avatar
Philip ABBET committed
160
161
    def __add__(self, other):
        """Adds two statistics data blocks"""
André Anjos's avatar
André Anjos committed
162

Philip ABBET's avatar
Philip ABBET committed
163
164
165
        retval = Statistics(copy.deepcopy(self._data))
        retval += other
        return retval
André Anjos's avatar
André Anjos committed
166
167


Philip ABBET's avatar
Philip ABBET committed
168
169
    def __iadd__(self, other):
        """Self-add statistics from another block"""
André Anjos's avatar
André Anjos committed
170

Philip ABBET's avatar
Philip ABBET committed
171
        if not isinstance(other, Statistics): return NotImplemented
André Anjos's avatar
André Anjos committed
172

Philip ABBET's avatar
Philip ABBET committed
173
174
        for key in ('user', 'system', 'total'):
            self._data['cpu'][key] += other._data['cpu'][key]
André Anjos's avatar
André Anjos committed
175

Philip ABBET's avatar
Philip ABBET committed
176
177
178
        for key in ('voluntary', 'involuntary'):
            self._data['cpu']['context_switches'][key] += \
                    other._data['cpu']['context_switches'][key]
André Anjos's avatar
André Anjos committed
179

Philip ABBET's avatar
Philip ABBET committed
180
181
182
        for key in ('rss', ): #gets the maximum between the two
            self._data['memory'][key] = max(other._data['memory'][key],
                self._data['memory'][key])
André Anjos's avatar
André Anjos committed
183

Philip ABBET's avatar
Philip ABBET committed
184
185
186
        for key in ('volume', 'blocks', 'time'):
            self._data['data'][key]['read'] += other._data['data'][key]['read']
            self._data['data'][key]['write'] += other._data['data'][key]['write']
André Anjos's avatar
André Anjos committed
187

Philip ABBET's avatar
Philip ABBET committed
188
        self._data['data']['files'] += other._data['data']['files']
André Anjos's avatar
André Anjos committed
189

Philip ABBET's avatar
Philip ABBET committed
190
191
        self._data['data']['network']['wait_time'] += \
                other._data['data']['network']['wait_time']
André Anjos's avatar
André Anjos committed
192

Philip ABBET's avatar
Philip ABBET committed
193
        return self
André Anjos's avatar
André Anjos committed
194
195


Philip ABBET's avatar
Philip ABBET committed
196
    def __str__(self):
André Anjos's avatar
André Anjos committed
197

Philip ABBET's avatar
Philip ABBET committed
198
        return self.as_json(2)
André Anjos's avatar
André Anjos committed
199
200


Philip ABBET's avatar
Philip ABBET committed
201
    def as_json(self, indent=None):
André Anjos's avatar
André Anjos committed
202

Philip ABBET's avatar
Philip ABBET committed
203
        return simplejson.dumps(self._data, indent=indent)
André Anjos's avatar
André Anjos committed
204
205


Philip ABBET's avatar
Philip ABBET committed
206
    def as_dict(self):
André Anjos's avatar
André Anjos committed
207

Philip ABBET's avatar
Philip ABBET committed
208
        return self._data
André Anjos's avatar
André Anjos committed
209
210


Philip ABBET's avatar
Philip ABBET committed
211
212
    def write(self, f):
        """Writes contents to a file-like object"""
André Anjos's avatar
André Anjos committed
213

Philip ABBET's avatar
Philip ABBET committed
214
215
216
        if hasattr(f, 'write'): f.write(str(self))
        else:
            with open(f, 'wt') as fobj: fobj.write(str(self))
André Anjos's avatar
André Anjos committed
217
218


219
#----------------------------------------------------------
André Anjos's avatar
André Anjos committed
220
221


222
def cpu_statistics(start, end):
Philip ABBET's avatar
Philip ABBET committed
223
    """Summarizes current CPU usage
André Anjos's avatar
André Anjos committed
224

Philip ABBET's avatar
Philip ABBET committed
225
226
227
228
    This method should be used when the currently set algorithm is the only one
    executed through the whole process. It is done for collecting resource
    statistics on separate processing environments. It follows the recipe in:
    http://stackoverflow.com/questions/30271942/get-docker-container-cpu-usage-as-percentage
André Anjos's avatar
André Anjos committed
229

Philip ABBET's avatar
Philip ABBET committed
230
    Returns:
André Anjos's avatar
André Anjos committed
231

Philip ABBET's avatar
Philip ABBET committed
232
      dict: A dictionary summarizing current CPU usage
André Anjos's avatar
André Anjos committed
233

Philip ABBET's avatar
Philip ABBET committed
234
    """
André Anjos's avatar
André Anjos committed
235

236
    if 'system_cpu_usage' not in end:
Philip ABBET's avatar
Philip ABBET committed
237
238
239
240
241
242
243
        return {
                'user': 0.0,
                'system': 0.0,
                'total': 0.0,
                'percent': 0.0,
                'processors': 1,
               }
Philip ABBET's avatar
Philip ABBET committed
244

Philip ABBET's avatar
Philip ABBET committed
245
246
247
248
    if start is not None:
        user_cpu = end['cpu_usage']['total_usage'] - \
            start['cpu_usage']['total_usage']
        total_cpu = end['system_cpu_usage'] - start['system_cpu_usage']
249

Philip ABBET's avatar
Philip ABBET committed
250
251
252
    else:
        user_cpu = end['cpu_usage']['total_usage']
        total_cpu = end['system_cpu_usage']
253

Philip ABBET's avatar
Philip ABBET committed
254
255
256
257
    user_cpu /= 1000000000. #in seconds
    total_cpu /= 1000000000. #in seconds
    processors = len(end['cpu_usage']['percpu_usage']) if \
        end['cpu_usage']['percpu_usage'] is not None else 1
258

Philip ABBET's avatar
Philip ABBET committed
259
260
261
262
263
264
265
    return {
            'user': user_cpu,
            'system': 0.,
            'total': total_cpu,
            'percent': 100.*processors*user_cpu/total_cpu if total_cpu else 0.,
            'processors': processors,
           }
André Anjos's avatar
André Anjos committed
266
267


268
269
270
#----------------------------------------------------------


271
def memory_statistics(data):
Philip ABBET's avatar
Philip ABBET committed
272
    """Summarizes current memory usage
André Anjos's avatar
André Anjos committed
273

Philip ABBET's avatar
Philip ABBET committed
274
275
276
    This method should be used when the currently set algorithm is the only one
    executed through the whole process. It is done for collecting resource
    statistics on separate processing environments.
André Anjos's avatar
André Anjos committed
277

Philip ABBET's avatar
Philip ABBET committed
278
    Returns:
André Anjos's avatar
André Anjos committed
279

Philip ABBET's avatar
Philip ABBET committed
280
      dict: A dictionary summarizing current memory usage
André Anjos's avatar
André Anjos committed
281

Philip ABBET's avatar
Philip ABBET committed
282
    """
André Anjos's avatar
André Anjos committed
283

Philip ABBET's avatar
Philip ABBET committed
284
285
    limit = float(data['limit'])
    memory = float(data['max_usage'])
André Anjos's avatar
André Anjos committed
286

Philip ABBET's avatar
Philip ABBET committed
287
288
289
290
291
    return {
            'rss': memory,
            'limit': limit,
            'percent': 100.*memory/limit if limit else 0.,
           }