diff --git a/pyblnet/blnet.py b/pyblnet/blnet.py index 8037f22..9308abd 100644 --- a/pyblnet/blnet.py +++ b/pyblnet/blnet.py @@ -68,7 +68,7 @@ def __init__( # especially not if not prefixed with http:// # see https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlparse host = urlparse(address).hostname or address - self.blnet_direct = BLNETDirect(host, ta_port) + self.blnet_direct = BLNETDirect(host, ta_port, timeout=timeout) def fetch(self, node=None): """ diff --git a/pyblnet/blnet_conn.py b/pyblnet/blnet_conn.py index 0037b05..f758412 100644 --- a/pyblnet/blnet_conn.py +++ b/pyblnet/blnet_conn.py @@ -10,6 +10,7 @@ """ from builtins import str, int from socket import socket, getaddrinfo, SOCK_STREAM, IPPROTO_TCP +import functools import struct from time import sleep from datetime import datetime @@ -27,11 +28,37 @@ END_READ = b"\xAD" RESET_DATA = b"\xAF" WAIT_TIME = 0xBA +# Seconds to wait when the BL-Net has no new data yet. Real devices answer +# GET_LATEST with "no data" for up to about 5 seconds after a read. +NO_DATA_WAIT = 2 MAX_RETRYS = 10 +# Length of a WAIT_TIME reply: 0xBA, seconds to wait, checksum +WAIT_SIZE = 3 DATASET_SIZE = 61 LATEST_SIZE = 56 +def _reset_on_error(method): + """ + Close the connection and forget the read state if a read fails, + so that the next call starts with a fresh connection. + Malformed replies are raised as ConnectionError. + """ + + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + try: + return method(self, *args, **kwargs) + except (struct.error, IndexError, ValueError) as ex: + self._reset() + raise ConnectionError(f"Invalid reply from BL-Net: {ex}") from ex + except BaseException: + self._reset() + raise + + return wrapper + + class BLNETDirect(object): """ A class for establishing a direct connection to the BLNET (rather than @@ -62,6 +89,7 @@ def __init__(self, address, port: int = 40000, reset: bool = False, timeout: flo self._count = None self._check_mode() + @_reset_on_error def get_count(self): """ Get the number of datasets in the bootloader memory @@ -128,20 +156,18 @@ def get_count(self): else: raise ConnectionError("Could not retreive count") + @_reset_on_error def _get_data(self, max_count=None): data = [] - try: - count = self._start_read() - if isinstance(max_count, int): - count = min(max_count, count) - for _ in range(0, count): - data.append(self._fetch_data()) - self._end_read(True) - return data - finally: - self._end_read(False) - return data + count = self._start_read() + if isinstance(max_count, int): + count = min(max_count, count) + for _ in range(0, count): + data.append(self._fetch_data()) + self._end_read(True) + return data + @_reset_on_error def _check_mode(self): """ Check if Bootloader Mode is supported @@ -170,7 +196,8 @@ def _connect(self): self._socket.settimeout(self._timeout) self._socket.connect(sockaddr) break - except: + except OSError: + self._socket.close() self._socket = None if self._socket is None: raise ConnectionError("Could not connect to BLNET") @@ -182,26 +209,67 @@ def _disconnect(self): self._socket.close() self._socket = None + def _reset(self): + """ + Close the connection if open and forget the read state + """ + if self._socket is not None: + self._disconnect() + self._count = None + self._address = None + def _query(self, command, length): """ Send a command to the bootloader and wait for the response if response is less then 32 bytes long return immediately @param commmand: string only ascii (needs byte length == string length) @param length: int length of response - @throws ConnectionError error when querying + @throws ConnectionError BL-Net closed the connection @return Binary: byte string """ - if len(command) == self._socket.send(command): - data = b"" - # get response until length or less than 32 bytes - while True: - data += self._socket.recv(length) - if len(data) <= 32 or len(data) >= length: - break - return data - - self._disconnect() - raise ConnectionError("Error while querying command {}".format(command)) + self._socket.sendall(command) + data = b"" + # get response until length or less than 32 bytes + while True: + received = self._socket.recv(length) + if not received: + raise ConnectionError("BL-Net closed the connection") + data += received + if len(data) <= 32 or len(data) >= length: + break + return data + + def _recv_exact(self, length): + """ + Receive exactly length bytes + @throws ConnectionError BL-Net closed the connection + @return Binary: byte string + """ + data = b"" + while len(data) < length: + received = self._socket.recv(length - len(data)) + if not received: + raise ConnectionError("BL-Net closed the connection") + data += received + return data + + def _query_latest(self, command): + """ + Send a GET_LATEST command and receive the complete reply. + The BL-Net may send a reply in several TCP segments, so the length + is derived from its first byte and the reply is read completely. + Reading only part of it would leave the rest in the socket and + shift every following reply by some bytes. + @return Binary: byte string, a "no data", "wait" or data reply + """ + self._socket.sendall(command) + first = self._recv_exact(1) + if first[0] == GET_LATEST: + # no new data available yet + return first + if first[0] == WAIT_TIME: + return first + self._recv_exact(WAIT_SIZE - 1) + return first + self._recv_exact(self._actual_size - 1) def _start_read(self): """ @@ -309,6 +377,7 @@ def _end_read(self, success=True): self._address = None self._disconnect() + @_reset_on_error def get_latest(self, max_retries=MAX_RETRYS): """ Fetch latest (current) data from the BLNet @@ -328,9 +397,12 @@ def get_latest(self, max_retries=MAX_RETRYS): # try 4 times sleeps = [] for n in range(0, max_retries): - data = self._query(command, self._actual_size) + data = self._query_latest(command) - if self._checksum(data): + if data == bytes([GET_LATEST]): + # no new data available yet ("Synchronisation" in TA's spec) + sleep(NO_DATA_WAIT) + elif self._checksum(data): binary = struct.unpack("<{}".format("B" * len(data)), data) if binary[0] == WAIT_TIME: @@ -343,6 +415,11 @@ def get_latest(self, max_retries=MAX_RETRYS): info["got"][frame] = n frames.update(self._split_latest(data, frame)) break + else: + # Unexpected bytes may still be on their way, reconnect + # so that they can't end up in the next reply + self._disconnect() + self._connect() else: # BL-Net did not send data for this frame within max_retries info["timeout"].append(frame) diff --git a/tests/test_direct.py b/tests/test_direct.py index fd1e25a..b8f5a0b 100644 --- a/tests/test_direct.py +++ b/tests/test_direct.py @@ -1,14 +1,21 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import gc import socket import threading +import time import unittest +import warnings +from unittest import mock from pyblnet import BLNET, BLNETDirect from tests.test_structure.blnet_direct_mock_server import ( BLNETDirectServer, DL_LATEST_RESPONSE, + GET_HEADER, + GET_LATEST, + READ_DATA, WAIT_RESPONSE, ) @@ -27,10 +34,11 @@ def tearDown(self): self.server.shutdown() self.server.server_close() - def blnet(self, max_retries=5): + def blnet(self, max_retries=5, timeout=5): return BLNET( ADDRESS, ta_port=self.port, + timeout=timeout, use_web=False, use_ta=True, max_retries=max_retries, @@ -53,10 +61,99 @@ def test_fetch_succeeds_on_last_retry(self): data = self.blnet(max_retries=3).fetch() self.assertEqual(data["analog"][6], {"value": 53.9}) + def test_fetch_reads_replies_split_into_segments(self): + self.server.split_latest = True + blnet = self.blnet() + for _ in range(3): + data = blnet.fetch() + self.assertEqual(data["analog"][6]["value"], 53.9) + self.assertEqual(data["energy"][1]["value"], 362847.4) + + def test_fetch_reconnects_after_invalid_reply(self): + # a stray byte ahead of the reply must not shift the following replies + self.server.latest_responses = [b"\x00" + DL_LATEST_RESPONSE, DL_LATEST_RESPONSE] + data = self.blnet().fetch() + self.assertEqual(data["analog"][6]["value"], 53.9) + self.assertEqual(data["energy"][1]["value"], 362847.4) + def test_timeout_does_not_change_global_socket_timeout(self): BLNETDirect(ADDRESS, self.port, timeout=7) self.assertIsNone(socket.getdefaulttimeout()) + def test_failed_connection_attempt_closes_socket(self): + # Port that refuses connections: bind it, then free it again + with socket.socket() as s: + s.bind((ADDRESS, 0)) + refused_port = s.getsockname()[1] + # The first address refuses the connection, the second one is the BL-NET + addresses = [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", port)) + for port in (refused_port, self.port) + ] + with mock.patch("pyblnet.blnet_conn.getaddrinfo", return_value=addresses): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", ResourceWarning) + BLNETDirect(ADDRESS, self.port, timeout=7) + gc.collect() + self.assertEqual([w for w in caught if w.category is ResourceWarning], []) + + def test_timeout_is_used_for_direct_connection(self): + self.server.silent = True + start = time.monotonic() + with self.assertRaises(socket.timeout): + BLNET(ADDRESS, ta_port=self.port, timeout=1, use_web=False, use_ta=True) + self.assertLess(time.monotonic() - start, 5) + + def test_fetch_recovers_after_timeout(self): + blnet = self.blnet(timeout=1) + self.server.silent = True + with self.assertRaises(socket.timeout): + blnet.fetch() + # BL-NET answers again, but the old connection is gone + self.server.silent = False + self.server.drop_connections() + self.assertEqual(blnet.fetch()["analog"][6], {"value": 53.9}) + + def test_fetch_raises_connection_error_when_blnet_hangs_up(self): + blnet = self.blnet(timeout=1) + self.server.close_on = GET_HEADER + with self.assertRaises(ConnectionError): + blnet.fetch() + + def test_fetch_recovers_after_blnet_hung_up(self): + blnet = self.blnet(timeout=1) + self.server.close_on = GET_LATEST + with self.assertRaises(ConnectionError): + blnet.fetch() + self.server.close_on = None + self.assertEqual(blnet.fetch()["analog"][6], {"value": 53.9}) + + def test_constructor_closes_socket_on_timeout(self): + self.server.silent = True + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", ResourceWarning) + with self.assertRaises(socket.timeout): + BLNETDirect(ADDRESS, self.port, timeout=1) + gc.collect() + self.assertEqual([w for w in caught if w.category is ResourceWarning], []) + + def test_read_logged_data_raises_when_blnet_hangs_up(self): + blnet_direct = BLNETDirect(ADDRESS, self.port, timeout=1) + self.server.close_on = READ_DATA + with self.assertRaises(ConnectionError): + blnet_direct._get_data(1) + + def test_read_logged_data_raises_on_invalid_reply(self): + blnet_direct = BLNETDirect(ADDRESS, self.port, timeout=1) + with self.assertRaises(ConnectionError): + blnet_direct._get_data(1) + + def test_fetch_waits_while_blnet_has_no_new_data(self): + # Real BL-NETs answer "no data" for up to a few seconds after a read + blnet = self.blnet(max_retries=3) + self.server.no_data_until = time.monotonic() + 1 + self.assertEqual(blnet.fetch()["analog"][6], {"value": 53.9}) + def test_fetch_raises_when_blnet_keeps_waiting(self): self.server.latest_responses = [WAIT_RESPONSE] with self.assertRaises(ConnectionError): diff --git a/tests/test_structure/blnet_direct_mock_server.py b/tests/test_structure/blnet_direct_mock_server.py index fbb96ab..29a523f 100644 --- a/tests/test_structure/blnet_direct_mock_server.py +++ b/tests/test_structure/blnet_direct_mock_server.py @@ -1,3 +1,5 @@ +import socket +import time from socketserver import BaseRequestHandler, ThreadingTCPServer # Responses recorded from a real BL-NET with one UVR1611 in DL mode @@ -9,10 +11,13 @@ ) # Asks the client to wait the given amount of seconds before trying again WAIT_RESPONSE = bytes.fromhex("ba00ba") +# No new data available yet ("Synchronisation" in TA's interface spec) +NO_DATA_RESPONSE = bytes.fromhex("ab") GET_MODE = 0x81 GET_HEADER = 0xAA GET_LATEST = 0xAB +READ_DATA = 0xAC END_READ = 0xAD @@ -25,15 +30,51 @@ def __init__(self, server_address): super().__init__(server_address, BLNETDirectRequestHandler) # Responses to GET_LATEST, the last one is repeated forever self.latest_responses = [DL_LATEST_RESPONSE] + # Accept connections but never answer + self.silent = False + # Close the connection when receiving this command + self.close_on = None + # Answer GET_LATEST with NO_DATA_RESPONSE until this time.monotonic() value + self.no_data_until = None + # Response to READ_DATA, by default one with a wrong checksum + self.read_data_response = bytes.fromhex("0001") + # Send the first byte of GET_LATEST replies in a TCP segment of its own, + # like real BL-NETs sometimes do + self.split_latest = False + # Currently open client connections + self.connections = set() + + def drop_connections(self): + """Close all open client connections, like a BL-NET that restarts""" + for connection in list(self.connections): + try: + connection.shutdown(socket.SHUT_RDWR) + except OSError: + # client closed it already (macOS raises ENOTCONN, Linux doesn't) + pass class BLNETDirectRequestHandler(BaseRequestHandler): def handle(self): + self.server.connections.add(self.request) + try: + self._handle_commands() + except OSError: + # connection was dropped + pass + finally: + self.server.connections.discard(self.request) + + def _handle_commands(self): while True: command = self.request.recv(1) if not command: return command = command[0] + if command == self.server.close_on: + return + if self.server.silent: + continue if command == GET_MODE: self.request.sendall(DL_MODE_RESPONSE) elif command == GET_HEADER: @@ -41,8 +82,21 @@ def handle(self): elif command == GET_LATEST: # frame number self.request.recv(1) + no_data_until = self.server.no_data_until + if no_data_until is not None and time.monotonic() < no_data_until: + self.request.sendall(NO_DATA_RESPONSE) + continue responses = self.server.latest_responses response = responses.pop(0) if len(responses) > 1 else responses[0] - self.request.sendall(response) + if self.server.split_latest and len(response) > 1: + self.request.sendall(response[:1]) + time.sleep(0.05) + self.request.sendall(response[1:]) + else: + self.request.sendall(response) + elif command == READ_DATA: + # address, number of frames, checksum + self.request.recv(5) + self.request.sendall(self.server.read_data_response) elif command == END_READ: self.request.sendall(bytes([END_READ]))