From d6faba00ad53d5359e1ca34245535ec7d0338630 Mon Sep 17 00:00:00 2001 From: Alexander Eisele Date: Sun, 11 Oct 2026 22:05:39 +0200 Subject: [PATCH] Replace htmldom with html.parser htmldom is unmaintained and mangles parts of the BL-NET pages (button text "<=" ends up as tag fragments in the extracted text). A small html.parser based helper extracts the text of matching elements the way the value regexes expect: entities like   kept as written and
as newline, including nested matching elements. Adds tests on the saved device pages for cases the mock server tests didn't cover (negative values, access denied page, blnet_test). They pass with both implementations, and both give identical results on all saved pages. requests is now the only runtime dependency. Co-Authored-By: Claude Opus 5.5 --- pyblnet/blnet_web.py | 98 +++++++++++++++++++++++++++++++---------- pyproject.toml | 1 - tests/test_web_pages.py | 61 +++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 24 deletions(-) create mode 100644 tests/test_web_pages.py diff --git a/pyblnet/blnet_web.py b/pyblnet/blnet_web.py index ed22e97..9f9d53b 100644 --- a/pyblnet/blnet_web.py +++ b/pyblnet/blnet_web.py @@ -10,10 +10,74 @@ """ import requests -from htmldom import htmldom import html import re from builtins import int +from html.parser import HTMLParser + + +class _ElementTextParser(HTMLParser): + """ + Collects the text of all elements with the given tag and class or id. + Entities like   are kept as written and
becomes a newline, + which is what the regular expressions below expect. + """ + + def __init__(self, tag, cls=None, id=None): + super().__init__(convert_charrefs=False) + self.tag = tag + self.cls = cls + self.id = id + # text of each matching element, in document order + self.texts = [] + # for each open element with self.tag: its index in self.texts, + # or None if it doesn't match. Matching elements can be nested. + self._open = [] + + def _matches(self, attrs): + attrs = dict(attrs) + return (self.cls is None or self.cls in (attrs.get("class") or "").split()) and ( + self.id is None or attrs.get("id") == self.id + ) + + def handle_starttag(self, tag, attrs): + if tag == self.tag: + if self._matches(attrs): + self.texts.append("") + self._open.append(len(self.texts) - 1) + else: + self._open.append(None) + elif tag == "br": + self.handle_data("\n") + + def handle_endtag(self, tag): + if tag == self.tag and self._open: + self._open.pop() + + def handle_data(self, data): + for index in self._open: + if index is not None: + self.texts[index] += data + + def handle_entityref(self, name): + self.handle_data(f"&{name};") + + def handle_charref(self, name): + self.handle_data(f"&#{name};") + + +def _element_texts(page, tag, cls=None, id=None): + """Texts of all elements in page with the given tag and class or id""" + parser = _ElementTextParser(tag, cls, id) + parser.feed(page) + parser.close() + return parser.texts + + +def _element_text(page, tag, cls=None, id=None): + """Text of the first such element, empty if there is none""" + texts = _element_texts(page, tag, cls, id) + return texts[0] if texts else "" def blnet_test(ip, timeout=5, id=0): @@ -28,13 +92,11 @@ def blnet_test(ip, timeout=5, id=0): r = requests.get(ip, timeout=timeout) except requests.exceptions.RequestException: return False - # Parse DOM object from HTMLCode - dom = htmldom.HtmlDom().createDom(r.text) # either a 'Zugriff verweigert' message is shown - if "BL-Net".lower() in dom.find("title").text().lower(): + if "BL-Net".lower() in _element_text(r.text, "title").lower(): return True # or (more often) the BL-NET Menü - if "BL-NET" in dom.find("div#head").text(): + if "BL-NET" in _element_text(r.text, "div", id="head"): return True return False @@ -183,18 +245,12 @@ def read_analog_values(self): ) except requests.exceptions.RequestException: return None - # Parse DOM object from HTMLCode - dom = htmldom.HtmlDom().createDom(r.text) - # Check if we didn't fail in access - # if 'BL-Net Zugang verweigert' in dom.find('title').text(): - # return None # get the element containing the interesting information - dom = dom.find("div.c")[1] + texts = _element_texts(r.text, "div", cls="c") # in case of access denied or other errors, return None - if dom is None: - return dom - # filter out the text - data_raw = dom.text() + if len(texts) < 2: + return None + data_raw = texts[1] # collect data in an array data = list() @@ -234,18 +290,14 @@ def read_digital_values(self): ) except requests.exceptions.RequestException: return None - # Parse DOM object from HTMLCode - dom = htmldom.HtmlDom().createDom(r.text) # Check if we didn't fail in access - if "BL-Net Zugang verweigert" in dom.find("title").text(): + if "BL-Net Zugang verweigert" in _element_text(r.text, "title"): return None # get the element containing the interesting information - dom = dom.find("div.c")[1] - if dom is None: + texts = _element_texts(r.text, "div", cls="c") + if len(texts) < 2: return None - - # filter out the text - data_raw = dom.text() + data_raw = texts[1] # collect data in an array data = list() diff --git a/pyproject.toml b/pyproject.toml index 227a115..060a6bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,6 @@ classifiers = [ "Programming Language :: Python :: 3.15" ] dependencies = [ - "htmldom", "requests", ] diff --git a/tests/test_web_pages.py b/tests/test_web_pages.py new file mode 100644 index 0000000..f533dae --- /dev/null +++ b/tests/test_web_pages.py @@ -0,0 +1,61 @@ +"""Parsing of saved BL-NET web pages, independent of the mock HTTP server""" + +import unittest +from pathlib import Path +from unittest import mock + +from pyblnet import BLNETWeb, blnet_test + +PAGES = Path(__file__).parent / "test_structure" + + +def page(name): + """Response serving a saved page, decoded like requests does for the BL-NET""" + return mock.Mock(text=(PAGES / name).read_bytes().decode("latin-1")) + + +def serve(name): + return mock.patch("pyblnet.blnet_web.requests.get", return_value=page(name)) + + +class BLNETWebPagesTest(unittest.TestCase): + def setUp(self): + with serve("main.htm"): + self.web = BLNETWeb("blnet", password=None) + + def test_blnet_test_recognizes_main_page(self): + with serve("main.htm"): + self.assertTrue(blnet_test("blnet")) + + def test_blnet_test_recognizes_access_denied_page(self): + with serve(".error.html"): + self.assertTrue(blnet_test("blnet")) + + def test_blnet_test_rejects_other_pages(self): + with serve("580500.htm"): + self.assertFalse(blnet_test("blnet")) + + def test_read_analog_values_with_negative_value(self): + with serve("580500.alternative.htm"): + values = self.web.read_analog_values() + self.assertEqual( + values, + [ + {"id": "1", "name": "TKollektor", "value": "-0.1", "unit_of_measurement": "°C"}, + {"id": "2", "name": "TSP.oben", "value": "47.1", "unit_of_measurement": "°C"}, + {"id": "3", "name": "TSP.unten", "value": "57.8", "unit_of_measurement": "°C"}, + {"id": "4", "name": "THeizkr.VL", "value": "45.8", "unit_of_measurement": "°C"}, + {"id": "5", "name": "Temp.Aussen", "value": "1.1", "unit_of_measurement": "°C"}, + {"id": "6", "name": "Temp.Raum", "value": "24.1", "unit_of_measurement": "°C"}, + {"id": "7", "name": "T Kaminofen", "value": "55.8", "unit_of_measurement": "°C"}, + {"id": "9", "name": "TZirku.RL", "value": "22.4", "unit_of_measurement": "°C"}, + ], + ) + + def test_read_analog_values_access_denied(self): + with serve(".error.html"): + self.assertIsNone(self.web.read_analog_values()) + + def test_read_digital_values_access_denied(self): + with serve(".error.html"): + self.assertIsNone(self.web.read_digital_values())