diff --git a/pyblnet/blnet.py b/pyblnet/blnet.py index 8037f22..b4d147b 100644 --- a/pyblnet/blnet.py +++ b/pyblnet/blnet.py @@ -12,10 +12,17 @@ """ from .blnet_web import BLNETWeb from .blnet_conn import BLNETDirect +from .blnet_parser import ANALOG_UNITS from urllib.parse import urlparse +# Units of the values only available via BLNETDirect, speed is a level 0-30 +UNITS = {"speed": None, "energy": "kWh", "power": "kW"} +# Analog input type for the units shown by the web interface +WEB_UNIT_TYPES = {unit: input_type for input_type, unit in ANALOG_UNITS.items() if unit is not None} + + class BLNET(object): """ General high-level BLNET interface. @@ -68,7 +75,7 @@ def __init__( # especially not if not prefixed with http:// # see https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlparse host = urlparse(address).hostname or address - self.blnet_direct = BLNETDirect(host, ta_port) + self.blnet_direct = BLNETDirect(host, ta_port, timeout=timeout) def fetch(self, node=None): """ @@ -88,6 +95,8 @@ def fetch(self, node=None): blnet_session.set_node(node) data["analog"] = self._convert_web(blnet_session.read_analog_values()) data["digital"] = self._convert_web(blnet_session.read_digital_values()) + for entry in data["analog"].values(): + entry["type"] = WEB_UNIT_TYPES.get(entry.get("unit_of_measurement")) if self.blnet_direct: direct = self.blnet_direct.get_latest(self.max_retries)[0] # Override values for analog and digital as values are @@ -100,12 +109,16 @@ def fetch(self, node=None): if value is None: continue data[domain][id] = {"value": value} + if domain == "analog": + input_type = direct["analog_types"][id] + data[domain][id]["type"] = input_type + data[domain][id]["unit_of_measurement"] = ANALOG_UNITS.get(input_type) for domain in ["speed", "energy", "power"]: for id, value in direct[domain].items(): if value is None: continue - data[domain][id] = {"value": value} + data[domain][id] = {"value": value, "unit_of_measurement": UNITS[domain]} return data def turn_on(self, digital_id, can_node=None): diff --git a/pyblnet/blnet_conn.py b/pyblnet/blnet_conn.py index 0037b05..f758412 100644 --- a/pyblnet/blnet_conn.py +++ b/pyblnet/blnet_conn.py @@ -10,6 +10,7 @@ """ from builtins import str, int from socket import socket, getaddrinfo, SOCK_STREAM, IPPROTO_TCP +import functools import struct from time import sleep from datetime import datetime @@ -27,11 +28,37 @@ END_READ = b"\xAD" RESET_DATA = b"\xAF" WAIT_TIME = 0xBA +# Seconds to wait when the BL-Net has no new data yet. Real devices answer +# GET_LATEST with "no data" for up to about 5 seconds after a read. +NO_DATA_WAIT = 2 MAX_RETRYS = 10 +# Length of a WAIT_TIME reply: 0xBA, seconds to wait, checksum +WAIT_SIZE = 3 DATASET_SIZE = 61 LATEST_SIZE = 56 +def _reset_on_error(method): + """ + Close the connection and forget the read state if a read fails, + so that the next call starts with a fresh connection. + Malformed replies are raised as ConnectionError. + """ + + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + try: + return method(self, *args, **kwargs) + except (struct.error, IndexError, ValueError) as ex: + self._reset() + raise ConnectionError(f"Invalid reply from BL-Net: {ex}") from ex + except BaseException: + self._reset() + raise + + return wrapper + + class BLNETDirect(object): """ A class for establishing a direct connection to the BLNET (rather than @@ -62,6 +89,7 @@ def __init__(self, address, port: int = 40000, reset: bool = False, timeout: flo self._count = None self._check_mode() + @_reset_on_error def get_count(self): """ Get the number of datasets in the bootloader memory @@ -128,20 +156,18 @@ def get_count(self): else: raise ConnectionError("Could not retreive count") + @_reset_on_error def _get_data(self, max_count=None): data = [] - try: - count = self._start_read() - if isinstance(max_count, int): - count = min(max_count, count) - for _ in range(0, count): - data.append(self._fetch_data()) - self._end_read(True) - return data - finally: - self._end_read(False) - return data + count = self._start_read() + if isinstance(max_count, int): + count = min(max_count, count) + for _ in range(0, count): + data.append(self._fetch_data()) + self._end_read(True) + return data + @_reset_on_error def _check_mode(self): """ Check if Bootloader Mode is supported @@ -170,7 +196,8 @@ def _connect(self): self._socket.settimeout(self._timeout) self._socket.connect(sockaddr) break - except: + except OSError: + self._socket.close() self._socket = None if self._socket is None: raise ConnectionError("Could not connect to BLNET") @@ -182,26 +209,67 @@ def _disconnect(self): self._socket.close() self._socket = None + def _reset(self): + """ + Close the connection if open and forget the read state + """ + if self._socket is not None: + self._disconnect() + self._count = None + self._address = None + def _query(self, command, length): """ Send a command to the bootloader and wait for the response if response is less then 32 bytes long return immediately @param commmand: string only ascii (needs byte length == string length) @param length: int length of response - @throws ConnectionError error when querying + @throws ConnectionError BL-Net closed the connection @return Binary: byte string """ - if len(command) == self._socket.send(command): - data = b"" - # get response until length or less than 32 bytes - while True: - data += self._socket.recv(length) - if len(data) <= 32 or len(data) >= length: - break - return data - - self._disconnect() - raise ConnectionError("Error while querying command {}".format(command)) + self._socket.sendall(command) + data = b"" + # get response until length or less than 32 bytes + while True: + received = self._socket.recv(length) + if not received: + raise ConnectionError("BL-Net closed the connection") + data += received + if len(data) <= 32 or len(data) >= length: + break + return data + + def _recv_exact(self, length): + """ + Receive exactly length bytes + @throws ConnectionError BL-Net closed the connection + @return Binary: byte string + """ + data = b"" + while len(data) < length: + received = self._socket.recv(length - len(data)) + if not received: + raise ConnectionError("BL-Net closed the connection") + data += received + return data + + def _query_latest(self, command): + """ + Send a GET_LATEST command and receive the complete reply. + The BL-Net may send a reply in several TCP segments, so the length + is derived from its first byte and the reply is read completely. + Reading only part of it would leave the rest in the socket and + shift every following reply by some bytes. + @return Binary: byte string, a "no data", "wait" or data reply + """ + self._socket.sendall(command) + first = self._recv_exact(1) + if first[0] == GET_LATEST: + # no new data available yet + return first + if first[0] == WAIT_TIME: + return first + self._recv_exact(WAIT_SIZE - 1) + return first + self._recv_exact(self._actual_size - 1) def _start_read(self): """ @@ -309,6 +377,7 @@ def _end_read(self, success=True): self._address = None self._disconnect() + @_reset_on_error def get_latest(self, max_retries=MAX_RETRYS): """ Fetch latest (current) data from the BLNet @@ -328,9 +397,12 @@ def get_latest(self, max_retries=MAX_RETRYS): # try 4 times sleeps = [] for n in range(0, max_retries): - data = self._query(command, self._actual_size) + data = self._query_latest(command) - if self._checksum(data): + if data == bytes([GET_LATEST]): + # no new data available yet ("Synchronisation" in TA's spec) + sleep(NO_DATA_WAIT) + elif self._checksum(data): binary = struct.unpack("<{}".format("B" * len(data)), data) if binary[0] == WAIT_TIME: @@ -343,6 +415,11 @@ def get_latest(self, max_retries=MAX_RETRYS): info["got"][frame] = n frames.update(self._split_latest(data, frame)) break + else: + # Unexpected bytes may still be on their way, reconnect + # so that they can't end up in the next reply + self._disconnect() + self._connect() else: # BL-Net did not send data for this frame within max_retries info["timeout"].append(frame) diff --git a/pyblnet/blnet_parser.py b/pyblnet/blnet_parser.py index c0d12c3..bca17d4 100644 --- a/pyblnet/blnet_parser.py +++ b/pyblnet/blnet_parser.py @@ -36,6 +36,22 @@ INT32_MASK = 0xFFFFFFFF INT32_SIGN = 0x80000000 +# Names of the analog input types, unused and undocumented types have none +ANALOG_TYPES = { + TYPE_DIGITAL: "digital", + TYPE_TEMP: "temperature", + TYPE_VOLUME: "volume_flow", + TYPE_RADIATION: "radiation", + TYPE_RAS: "temperature", +} +# Units of the analog input types, as the web interface shows them +ANALOG_UNITS = { + "digital": None, + "temperature": "°C", + "volume_flow": "l/h", + "radiation": "W/m²", +} + class BLNETParser: def __init__(self, data): @@ -73,7 +89,9 @@ def __init__(self, data): analog = struct.unpack("<{}{}".format("H" * 16, "x" * (len(data) - 32)), data) self.analog = {} + self.analog_types = {} for channel in range(0, 16): + self.analog_types[channel + 1] = ANALOG_TYPES.get(analog[channel] & TYPE_MASK) value = self._convert_analog(analog[channel]) self.analog[channel + 1] = None if value is None else round(value, 3) diff --git a/tests/test_direct.py b/tests/test_direct.py index fd1e25a..772b6c4 100644 --- a/tests/test_direct.py +++ b/tests/test_direct.py @@ -1,14 +1,21 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import gc import socket import threading +import time import unittest +import warnings +from unittest import mock from pyblnet import BLNET, BLNETDirect from tests.test_structure.blnet_direct_mock_server import ( BLNETDirectServer, DL_LATEST_RESPONSE, + GET_HEADER, + GET_LATEST, + READ_DATA, WAIT_RESPONSE, ) @@ -27,10 +34,11 @@ def tearDown(self): self.server.shutdown() self.server.server_close() - def blnet(self, max_retries=5): + def blnet(self, max_retries=5, timeout=5): return BLNET( ADDRESS, ta_port=self.port, + timeout=timeout, use_web=False, use_ta=True, max_retries=max_retries, @@ -38,25 +46,123 @@ def blnet(self, max_retries=5): def test_fetch(self): data = self.blnet().fetch() - self.assertEqual(data["analog"][6], {"value": 53.9}) + self.assertEqual( + data["analog"][6], + {"value": 53.9, "type": "temperature", "unit_of_measurement": "°C"}, + ) + self.assertEqual( + data["analog"][16], + {"value": 0, "type": "volume_flow", "unit_of_measurement": "l/h"}, + ) self.assertEqual(data["digital"][1], {"value": 1}) - self.assertEqual(data["speed"][1], {"value": 24}) - self.assertEqual(data["energy"], {1: {"value": 362847.4}}) + # speed level 0-30, no unit + self.assertEqual(data["speed"][1], {"value": 24, "unit_of_measurement": None}) + self.assertEqual(data["energy"], {1: {"value": 362847.4, "unit_of_measurement": "kWh"}}) + self.assertEqual(data["power"], {1: {"value": 0.0, "unit_of_measurement": "kW"}}) def test_get_value_by_id(self): value, entry, _ = self.blnet().get_analog_value(id=6) self.assertEqual(value, 53.9) - self.assertEqual(entry, {"value": 53.9}) + self.assertEqual(entry["value"], 53.9) def test_fetch_succeeds_on_last_retry(self): self.server.latest_responses = [WAIT_RESPONSE, WAIT_RESPONSE, DL_LATEST_RESPONSE] data = self.blnet(max_retries=3).fetch() - self.assertEqual(data["analog"][6], {"value": 53.9}) + self.assertEqual(data["analog"][6]["value"], 53.9) + + def test_fetch_reads_replies_split_into_segments(self): + self.server.split_latest = True + blnet = self.blnet() + for _ in range(3): + data = blnet.fetch() + self.assertEqual(data["analog"][6]["value"], 53.9) + self.assertEqual(data["energy"][1]["value"], 362847.4) + + def test_fetch_reconnects_after_invalid_reply(self): + # a stray byte ahead of the reply must not shift the following replies + self.server.latest_responses = [b"\x00" + DL_LATEST_RESPONSE, DL_LATEST_RESPONSE] + data = self.blnet().fetch() + self.assertEqual(data["analog"][6]["value"], 53.9) + self.assertEqual(data["energy"][1]["value"], 362847.4) def test_timeout_does_not_change_global_socket_timeout(self): BLNETDirect(ADDRESS, self.port, timeout=7) self.assertIsNone(socket.getdefaulttimeout()) + def test_failed_connection_attempt_closes_socket(self): + # Port that refuses connections: bind it, then free it again + with socket.socket() as s: + s.bind((ADDRESS, 0)) + refused_port = s.getsockname()[1] + # The first address refuses the connection, the second one is the BL-NET + addresses = [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", port)) + for port in (refused_port, self.port) + ] + with mock.patch("pyblnet.blnet_conn.getaddrinfo", return_value=addresses): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", ResourceWarning) + BLNETDirect(ADDRESS, self.port, timeout=7) + gc.collect() + self.assertEqual([w for w in caught if w.category is ResourceWarning], []) + + def test_timeout_is_used_for_direct_connection(self): + self.server.silent = True + start = time.monotonic() + with self.assertRaises(socket.timeout): + BLNET(ADDRESS, ta_port=self.port, timeout=1, use_web=False, use_ta=True) + self.assertLess(time.monotonic() - start, 5) + + def test_fetch_recovers_after_timeout(self): + blnet = self.blnet(timeout=1) + self.server.silent = True + with self.assertRaises(socket.timeout): + blnet.fetch() + # BL-NET answers again, but the old connection is gone + self.server.silent = False + self.server.drop_connections() + self.assertEqual(blnet.fetch()["analog"][6]["value"], 53.9) + + def test_fetch_raises_connection_error_when_blnet_hangs_up(self): + blnet = self.blnet(timeout=1) + self.server.close_on = GET_HEADER + with self.assertRaises(ConnectionError): + blnet.fetch() + + def test_fetch_recovers_after_blnet_hung_up(self): + blnet = self.blnet(timeout=1) + self.server.close_on = GET_LATEST + with self.assertRaises(ConnectionError): + blnet.fetch() + self.server.close_on = None + self.assertEqual(blnet.fetch()["analog"][6]["value"], 53.9) + + def test_constructor_closes_socket_on_timeout(self): + self.server.silent = True + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", ResourceWarning) + with self.assertRaises(socket.timeout): + BLNETDirect(ADDRESS, self.port, timeout=1) + gc.collect() + self.assertEqual([w for w in caught if w.category is ResourceWarning], []) + + def test_read_logged_data_raises_when_blnet_hangs_up(self): + blnet_direct = BLNETDirect(ADDRESS, self.port, timeout=1) + self.server.close_on = READ_DATA + with self.assertRaises(ConnectionError): + blnet_direct._get_data(1) + + def test_read_logged_data_raises_on_invalid_reply(self): + blnet_direct = BLNETDirect(ADDRESS, self.port, timeout=1) + with self.assertRaises(ConnectionError): + blnet_direct._get_data(1) + + def test_fetch_waits_while_blnet_has_no_new_data(self): + # Real BL-NETs answer "no data" for up to a few seconds after a read + blnet = self.blnet(max_retries=3) + self.server.no_data_until = time.monotonic() + 1 + self.assertEqual(blnet.fetch()["analog"][6]["value"], 53.9) + def test_fetch_raises_when_blnet_keeps_waiting(self): self.server.latest_responses = [WAIT_RESPONSE] with self.assertRaises(ConnectionError): diff --git a/tests/test_parser.py b/tests/test_parser.py index 1e019e7..c90e12a 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -43,6 +43,16 @@ def test_parser_disconnected_temperature_sensor_is_none(self): assert parsed_values['analog'][1] is None assert parsed_values['analog'][2] == 409.4 + def test_parser_input_types(self): + # temperature, volume flow, radiation, digital, unused, room sensor, undocumented type 101 + inputs = [0x2000, 0x3000, 0x4000, 0x1000, 0x0000, 0x7000, 0x5000] + [0x2000] * 9 + data = struct.pack('<16H', *inputs) + bytes(55 - 32) + parsed_values = BLNETParser(data).to_dict() + + expected_types = {1: 'temperature', 2: 'volume_flow', 3: 'radiation', 4: 'digital', 5: None, 6: 'temperature', 7: None} + expected_types.update({channel: 'temperature' for channel in range(8, 17)}) + assert parsed_values['analog_types'] == expected_types + def test_parser_heat_meter_1_active(self): # Latest data of a real UVR1611 in DL mode (header and checksum stripped), # only heat meter 1 is active (active byte = 0x01) diff --git a/tests/test_structure/blnet_direct_mock_server.py b/tests/test_structure/blnet_direct_mock_server.py index fbb96ab..29a523f 100644 --- a/tests/test_structure/blnet_direct_mock_server.py +++ b/tests/test_structure/blnet_direct_mock_server.py @@ -1,3 +1,5 @@ +import socket +import time from socketserver import BaseRequestHandler, ThreadingTCPServer # Responses recorded from a real BL-NET with one UVR1611 in DL mode @@ -9,10 +11,13 @@ ) # Asks the client to wait the given amount of seconds before trying again WAIT_RESPONSE = bytes.fromhex("ba00ba") +# No new data available yet ("Synchronisation" in TA's interface spec) +NO_DATA_RESPONSE = bytes.fromhex("ab") GET_MODE = 0x81 GET_HEADER = 0xAA GET_LATEST = 0xAB +READ_DATA = 0xAC END_READ = 0xAD @@ -25,15 +30,51 @@ def __init__(self, server_address): super().__init__(server_address, BLNETDirectRequestHandler) # Responses to GET_LATEST, the last one is repeated forever self.latest_responses = [DL_LATEST_RESPONSE] + # Accept connections but never answer + self.silent = False + # Close the connection when receiving this command + self.close_on = None + # Answer GET_LATEST with NO_DATA_RESPONSE until this time.monotonic() value + self.no_data_until = None + # Response to READ_DATA, by default one with a wrong checksum + self.read_data_response = bytes.fromhex("0001") + # Send the first byte of GET_LATEST replies in a TCP segment of its own, + # like real BL-NETs sometimes do + self.split_latest = False + # Currently open client connections + self.connections = set() + + def drop_connections(self): + """Close all open client connections, like a BL-NET that restarts""" + for connection in list(self.connections): + try: + connection.shutdown(socket.SHUT_RDWR) + except OSError: + # client closed it already (macOS raises ENOTCONN, Linux doesn't) + pass class BLNETDirectRequestHandler(BaseRequestHandler): def handle(self): + self.server.connections.add(self.request) + try: + self._handle_commands() + except OSError: + # connection was dropped + pass + finally: + self.server.connections.discard(self.request) + + def _handle_commands(self): while True: command = self.request.recv(1) if not command: return command = command[0] + if command == self.server.close_on: + return + if self.server.silent: + continue if command == GET_MODE: self.request.sendall(DL_MODE_RESPONSE) elif command == GET_HEADER: @@ -41,8 +82,21 @@ def handle(self): elif command == GET_LATEST: # frame number self.request.recv(1) + no_data_until = self.server.no_data_until + if no_data_until is not None and time.monotonic() < no_data_until: + self.request.sendall(NO_DATA_RESPONSE) + continue responses = self.server.latest_responses response = responses.pop(0) if len(responses) > 1 else responses[0] - self.request.sendall(response) + if self.server.split_latest and len(response) > 1: + self.request.sendall(response[:1]) + time.sleep(0.05) + self.request.sendall(response[1:]) + else: + self.request.sendall(response) + elif command == READ_DATA: + # address, number of frames, checksum + self.request.recv(5) + self.request.sendall(self.server.read_data_response) elif command == END_READ: self.request.sendall(bytes([END_READ])) diff --git a/tests/web_raw/web_state.py b/tests/web_raw/web_state.py index fea4126..ea1b382 100644 --- a/tests/web_raw/web_state.py +++ b/tests/web_raw/web_state.py @@ -3,48 +3,56 @@ 1: { "id": "1", "name": "TKollektor", + "type": "temperature", "unit_of_measurement": "°C", "value": "5.3", }, 2: { "id": "2", "name": "TSP.oben", + "type": "temperature", "unit_of_measurement": "°C", "value": "46.3", }, 3: { "id": "3", "name": "TSP.unten", + "type": "temperature", "unit_of_measurement": "°C", "value": "53.1", }, 4: { "id": "4", "name": "THeizkr.VL", + "type": "temperature", "unit_of_measurement": "°C", "value": "44.8", }, 5: { "id": "5", "name": "Temp.Aussen", + "type": "temperature", "unit_of_measurement": "°C", "value": "-72.3", }, 6: { "id": "6", "name": "Temp.Raum", + "type": "temperature", "unit_of_measurement": "°C", "value": "24.6", }, 7: { "id": "7", "name": "T Kaminofen", + "type": "temperature", "unit_of_measurement": "°C", "value": "20.9", }, 9: { "id": "9", "name": "TZirku.RL", + "type": "temperature", "unit_of_measurement": "°C", "value": "36.6", }, @@ -68,42 +76,49 @@ "id": "2", "name": "TSP.oben", "value": "47.1", + "type": "temperature", "unit_of_measurement": "°C", }, 3: { "id": "3", "name": "TSP.unten", "value": "57.8", + "type": "temperature", "unit_of_measurement": "°C", }, 4: { "id": "4", "name": "THeizkr.VL", "value": "45.8", + "type": "temperature", "unit_of_measurement": "°C", }, 5: { "id": "5", "name": "Temp.Aussen", "value": "1.1", + "type": "temperature", "unit_of_measurement": "°C", }, 6: { "id": "6", "name": "Temp.Raum", "value": "24.1", + "type": "temperature", "unit_of_measurement": "°C", }, 7: { "id": "7", "name": "T Kaminofen", "value": "55.8", + "type": "temperature", "unit_of_measurement": "°C", }, 9: { "id": "9", "name": "TZirku.RL", "value": "22.4", + "type": "temperature", "unit_of_measurement": "°C", }, },