Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
beat
beat.backend.python
Commits
41726a0b
Commit
41726a0b
authored
Oct 24, 2017
by
Philip ABBET
Browse files
Optimize the number of messages sent between the remote inputs and the message handler
parent
3e7fdebc
Changes
2
Hide whitespace changes
Inline
Side-by-side
beat/backend/python/inputs.py
View file @
41726a0b
...
...
@@ -235,31 +235,35 @@ class RemoteInput(BaseInput):
def
__init__
(
self
,
name
,
data_format
,
socket
,
unpack
=
True
):
super
(
RemoteInput
,
self
).
__init__
(
name
,
data_format
)
self
.
socket
=
socket
self
.
comm_time
=
0.0
# Total time spent on communication
self
.
_unpack
=
unpack
self
.
socket
=
socket
self
.
comm_time
=
0.0
# Total time spent on communication
self
.
_unpack
=
unpack
self
.
_has_more_data
=
None
# To avoid repetitive requests
def
hasMoreData
(
self
):
"""Indicates if there is more data to process on the input"""
logger
.
debug
(
'send: (hmd) has-more-data %s %s'
,
self
.
group
.
channel
,
self
.
name
)
if
self
.
_has_more_data
is
None
:
logger
.
debug
(
'send: (hmd) has-more-data %s %s'
,
self
.
group
.
channel
,
self
.
name
)
_start
=
time
.
time
()
_start
=
time
.
time
()
self
.
socket
.
send
(
'hmd'
,
zmq
.
SNDMORE
)
self
.
socket
.
send
(
self
.
group
.
channel
,
zmq
.
SNDMORE
)
self
.
socket
.
send
(
self
.
name
)
self
.
socket
.
send
(
'hmd'
,
zmq
.
SNDMORE
)
self
.
socket
.
send
(
self
.
group
.
channel
,
zmq
.
SNDMORE
)
self
.
socket
.
send
(
self
.
name
)
answer
=
self
.
socket
.
recv
()
answer
=
self
.
socket
.
recv
()
self
.
comm_time
+=
time
.
time
()
-
_start
logger
.
debug
(
'recv: %s'
,
answer
)
self
.
comm_time
+=
time
.
time
()
-
_start
logger
.
debug
(
'recv: %s'
,
answer
)
if
answer
==
'err'
:
process_error
(
self
.
socket
)
if
answer
==
'err'
:
process_error
(
self
.
socket
)
return
answer
==
'tru'
self
.
_has_more_data
=
(
answer
==
'tru'
)
return
self
.
_has_more_data
def
next
(
self
):
...
...
@@ -287,6 +291,7 @@ class RemoteInput(BaseInput):
self
.
nb_data_blocks_read
+=
1
self
.
data_same_as_previous
=
False
self
.
_has_more_data
=
None
def
unpack
(
self
,
packed
):
...
...
@@ -437,22 +442,11 @@ class InputGroup:
if
self
.
socket
is
None
:
return
False
logger
.
debug
(
'send: (hmd) has-more-data %s'
,
self
.
channel
)
for
x
in
self
.
remoteInputs
():
if
x
.
hasMoreData
():
return
True
_start
=
time
.
time
()
self
.
socket
.
send
(
'hmd'
,
zmq
.
SNDMORE
)
self
.
socket
.
send
(
self
.
channel
)
answer
=
self
.
socket
.
recv
()
self
.
comm_time
+=
time
.
time
()
-
_start
logger
.
debug
(
'recv: %s'
,
answer
)
if
answer
==
'err'
:
process_error
(
self
.
socket
)
return
answer
==
'tru'
return
False
def
next
(
self
):
...
...
@@ -471,42 +465,10 @@ class InputGroup:
for
input
in
[
x
for
x
in
inputs_to_update
if
isinstance
(
x
,
Input
)
]:
input
.
next
()
input
.
data_same_as_previous
=
False
remote_inputs_to_update
=
list
([
x
for
x
in
inputs_to_update
if
isinstance
(
x
,
RemoteInput
)
])
if
len
(
remote_inputs_to_update
)
>
0
:
logger
.
debug
(
'send: (nxt) next %s'
,
self
.
channel
)
self
.
socket
.
send
(
'nxt'
,
zmq
.
SNDMORE
)
self
.
socket
.
send
(
self
.
channel
)
# read all incomming data
_start
=
time
.
time
()
more
=
True
parts
=
[]
while
more
:
parts
.
append
(
self
.
socket
.
recv
())
if
parts
[
-
1
]
==
'err'
:
self
.
comm_time
+=
time
.
time
()
-
_start
process_error
(
self
.
socket
)
more
=
self
.
socket
.
getsockopt
(
zmq
.
RCVMORE
)
n
=
int
(
parts
.
pop
(
0
))
logger
.
debug
(
'recv: %d (inputs)'
,
n
)
for
k
in
range
(
n
):
name
=
parts
.
pop
(
0
)
logger
.
debug
(
'recv: %s (data follows)'
,
name
)
inpt
=
self
[
name
]
if
inpt
is
None
:
raise
RuntimeError
(
"Could not find input `%s' at input group for "
\
"channel `%s' while performing `next' operation on this group "
\
"(current reading position is %d/%d)"
%
\
(
name
,
self
.
channel
,
k
,
n
))
inpt
.
data_index
=
int
(
parts
.
pop
(
0
))
inpt
.
data_index_end
=
int
(
parts
.
pop
(
0
))
inpt
.
unpack
(
parts
.
pop
(
0
))
inpt
.
nb_data_blocks_read
+=
1
self
.
comm_time
+=
time
.
time
()
-
_start
for
remote_input
in
remote_inputs_to_update
:
remote_input
.
next
()
for
input
in
inputs_up_to_date
:
input
.
data_same_as_previous
=
True
...
...
beat/backend/python/message_handler.py
View file @
41726a0b
...
...
@@ -105,8 +105,6 @@ class MessageHandler(threading.Thread):
more
=
self
.
socket
.
getsockopt
(
zmq
.
RCVMORE
)
command
=
parts
[
0
]
logger
.
debug
(
"recv: %s"
,
command
)
if
command
in
self
.
callbacks
:
try
:
#to handle command
self
.
callbacks
[
command
](
*
parts
[
1
:])
...
...
@@ -147,6 +145,8 @@ class MessageHandler(threading.Thread):
break
else
:
logger
.
debug
(
"recv: %s"
,
command
)
message
=
"Command `%s' is not implemented - stopping user process"
\
%
command
logger
.
error
(
message
)
...
...
@@ -170,81 +170,49 @@ class MessageHandler(threading.Thread):
return
retval
def
next
(
self
,
channel
,
name
=
None
):
"""Syntax: nxt channel [name] ..."""
if
name
is
not
None
:
#single input
logger
.
debug
(
'recv: nxt %s %s'
,
channel
,
name
)
input_candidate
=
self
.
_get_input_candidate
(
channel
,
name
)
input_candidate
.
next
()
if
input_candidate
.
data
is
None
:
#error
message
=
"User algorithm asked for more data for channel "
\
"`%s' on input `%s', but it is over (no more data). This "
\
"normally indicates a programming error on the user "
\
"side."
%
(
channel
,
name
)
self
.
user_error
+=
message
+
'
\n
'
raise
RuntimeError
(
message
)
if
isinstance
(
input_candidate
.
data
,
baseformat
.
baseformat
):
packed
=
input_candidate
.
data
.
pack
()
else
:
packed
=
input_candidate
.
data
logger
.
debug
(
'send: <bin> (size=%d), indexes=(%d, %d)'
,
len
(
packed
),
input_candidate
.
data_index
,
input_candidate
.
data_index_end
)
self
.
socket
.
send
(
'%d'
%
input_candidate
.
data_index
,
zmq
.
SNDMORE
)
self
.
socket
.
send
(
'%d'
%
input_candidate
.
data_index_end
,
zmq
.
SNDMORE
)
self
.
socket
.
send
(
packed
)
else
:
#whole group data
logger
.
debug
(
'recv: nxt %s'
,
channel
)
channel_group
=
self
.
input_list
.
group
(
channel
)
# Call next() on the group
channel_group
.
restricted_access
=
False
channel_group
.
next
()
channel_group
.
restricted_access
=
True
# Loop over the inputs
inputs_to_go
=
len
(
channel_group
)
self
.
socket
.
send
(
str
(
inputs_to_go
),
zmq
.
SNDMORE
)
for
inp
in
channel_group
:
logger
.
debug
(
'send: %s'
,
inp
.
name
)
self
.
socket
.
send
(
str
(
inp
.
name
),
zmq
.
SNDMORE
)
if
inp
.
data
is
None
:
message
=
"User algorithm process asked for more data on channel "
\
"`%s' (all inputs), but input `%s' has nothing. This "
\
"normally indicates a programming error on the user "
\
"side."
%
(
channel
,
inp
.
name
)
self
.
user_error
+=
message
+
'
\n
'
raise
RuntimeError
(
message
)
elif
isinstance
(
inp
.
data
,
baseformat
.
baseformat
):
packed
=
inp
.
data
.
pack
()
else
:
packed
=
inp
.
data
logger
.
debug
(
'send: <bin> (size=%d), indexes=(%d, %d)'
,
len
(
packed
),
inp
.
data_index
,
inp
.
data_index_end
)
self
.
socket
.
send
(
'%d'
%
inp
.
data_index
,
zmq
.
SNDMORE
)
self
.
socket
.
send
(
'%d'
%
inp
.
data_index_end
,
zmq
.
SNDMORE
)
inputs_to_go
-=
1
if
inputs_to_go
>
0
:
self
.
socket
.
send
(
packed
,
zmq
.
SNDMORE
)
else
:
self
.
socket
.
send
(
packed
)
def
next
(
self
,
channel
,
name
):
"""Syntax: nxt channel name ..."""
logger
.
debug
(
'recv: nxt %s %s'
,
channel
,
name
)
channel_group
=
self
.
input_list
.
group
(
channel
)
restricted
=
channel_group
.
restricted_access
channel_group
.
restricted_access
=
False
input_candidate
=
self
.
_get_input_candidate
(
channel
,
name
)
input_candidate
.
next
()
channel_group
.
restricted_access
=
restricted
if
input_candidate
.
data
is
None
:
#error
message
=
"User algorithm asked for more data for channel "
\
"`%s' on input `%s', but it is over (no more data). This "
\
"normally indicates a programming error on the user "
\
"side."
%
(
channel
,
name
)
self
.
user_error
+=
message
+
'
\n
'
raise
RuntimeError
(
message
)
if
isinstance
(
input_candidate
.
data
,
baseformat
.
baseformat
):
packed
=
input_candidate
.
data
.
pack
()
else
:
packed
=
input_candidate
.
data
logger
.
debug
(
'send: <bin> (size=%d), indexes=(%d, %d)'
,
len
(
packed
),
input_candidate
.
data_index
,
input_candidate
.
data_index_end
)
self
.
socket
.
send
(
'%d'
%
input_candidate
.
data_index
,
zmq
.
SNDMORE
)
self
.
socket
.
send
(
'%d'
%
input_candidate
.
data_index_end
,
zmq
.
SNDMORE
)
self
.
socket
.
send
(
packed
)
def
has_more_data
(
self
,
channel
,
name
=
None
):
"""Syntax: hmd channel [name]"""
if
name
:
#single input
logger
.
debug
(
'recv: hmd %s %s'
,
channel
,
name
)
input_candidate
=
self
.
_get_input_candidate
(
channel
,
name
)
what
=
'tru'
if
input_candidate
.
hasMoreData
()
else
'fal'
def
has_more_data
(
self
,
channel
,
name
):
"""Syntax: hmd channel name"""
else
:
#for all channel names
logger
.
debug
(
'recv: hmd %s'
,
channel
)
channel_group
=
self
.
input_list
.
group
(
channel
)
what
=
'tru'
if
channel_group
.
hasMoreData
()
else
'fal'
logger
.
debug
(
'recv: hmd %s %s'
,
channel
,
name
)
input_candidate
=
self
.
_get_input_candidate
(
channel
,
name
)
what
=
'tru'
if
input_candidate
.
hasMoreData
()
else
'fal'
logger
.
debug
(
'send: %s'
,
what
)
self
.
socket
.
send
(
what
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment