fix: robust XML request handling

README.md

@@ -211,6 +211,17 @@ timestamp: true # false by default
 
 To access the URL for data collection, send a GET request (or navigate) to [http://localhost:8080/measure](http://localhost:8080/measure).
 
+To store data, send a POST request to `/measure` with the values in one of the following ways:
+
+- query parameters, e.g. `POST /measure?temperature=21.5&humidity=40` (works with every format),
+- a JSON object body, e.g. `{"temperature": 21.5, "humidity": 40}` (works with every format),
+- an XML body (with `format='xml'`), where each child of the root element is one value; the name of the root element is not checked:
+  ```sh
+  curl -X POST http://localhost:8080/measure -H "Content-Type: application/xml" -d '<data><temperature>21.5</temperature><humidity>40</humidity></data>'
+  ```
+
+A malformed XML body or a JSON body that is not an object is rejected with a `400` response. With `format='xml'`, column names must be valid XML element names: letters, digits, `_`, `-` or `.`, starting with a letter or `_`, without spaces, and not starting with `xml`.
+
 To restrict access to the collected data, define the following setting in the `configuration.yml` file in the root directory:
 ```yml
 password: 'password' # Password for data access
@@ -220,7 +231,7 @@ The password can be sent with any request (`/measure`, `/data` and `/export`) in
 
 - `Authorization: Bearer password` header (recommended),
 - `X-API-Key: password` header,
-- `"password": "password"` field in a JSON body (`/measure` only),
+- `"password": "password"` field in a JSON body or `<password>password</password>` element in an XML body (`/measure` only),
 - `?password=password` query parameter (supported for backward compatibility).
 
 Prefer one of the headers, since URLs (including query parameters) are often written to server and proxy logs. For example:

docs/getting_started.rst

@@ -83,6 +83,18 @@ To store data collection timestamps, define the following setting in the ``confi
 
 To access the URL for data collection, send a GET request (or navigate) to `http://localhost:8080/measure <http://localhost:8080/measure>`_.
 
+To store data, send a POST request to ``/measure`` with the values in one of the following ways:
+
+- query parameters, e.g. ``POST /measure?temperature=21.5&humidity=40`` (works with every format),
+- a JSON object body, e.g. ``{"temperature": 21.5, "humidity": 40}`` (works with every format),
+- an XML body (with ``format='xml'``), where each child of the root element is one value; the name of the root element is not checked:
+
+.. code:: sh
+
+    curl -X POST http://localhost:8080/measure -H "Content-Type: application/xml" -d '<data><temperature>21.5</temperature><humidity>40</humidity></data>'
+
+A malformed XML body or a JSON body that is not an object is rejected with a ``400`` response. With ``format='xml'``, column names must be valid XML element names: letters, digits, ``_``, ``-`` or ``.``, starting with a letter or ``_``, without spaces, and not starting with ``xml``.
+
 To restrict access to the collected data, define the following setting in the ``configuration.yml`` file in the root directory:
 
 .. code:: yaml
@@ -93,7 +105,7 @@ The password can be sent with any request (``/measure``, ``/data`` and ``/export
 
 - ``Authorization: Bearer password`` header (recommended),
 - ``X-API-Key: password`` header,
-- ``"password": "password"`` field in a JSON body (``/measure`` only),
+- ``"password": "password"`` field in a JSON body or ``<password>password</password>`` element in an XML body (``/measure`` only),
 - ``?password=password`` query parameter (supported for backward compatibility).
 
 Prefer one of the headers, since URLs (including query parameters) are often written to server and proxy logs. For example:

docs/installation.rst

@@ -26,27 +26,27 @@ List of succulent's dependencies:
 +----------------------+----------------------+
 | Package              | Version              |
 +======================+======================+
-| pandas               | ^2.0.1               |
+| pandas               | >=2.0.1,<3.0.0       |
 +----------------------+----------------------+
-| pyyaml               | ^6.0                 |
+| pyyaml               | >=6.0,<7.0           |
 +----------------------+----------------------+
-| flask                | ^2.3.2               |
+| flask                | >=2.3.2,<4.0.0       |
 +----------------------+----------------------+
-| xmltodict            | ^0.13.0              |
+| lxml                 | >=5.2.2,<7.0.0       |
 +----------------------+----------------------+
-| lxml                 | ^5.1.0               |
+| numpy                | >=1.26.0,<3.0.0      |
 +----------------------+----------------------+
 
 List of succulent's development dependencies:
 
 +----------------------+------------+
 | Package              | Version    |
 +======================+============+
-| pytest               | ^6.2       |
+| pytest               | ^9.0.3     |
 +----------------------+------------+
-| sphinx               | ^4.4.0     |
+| sphinx               | >=7.0      |
 +----------------------+------------+
-| sphinx-rtd-theme     | ^1.0.0     |
+| sphinx-rtd-theme     | >=2.0      |
 +----------------------+------------+
-| sphinxcontrib-bibtex | ^2.4.1     |
+| sphinxcontrib-bibtex | ^2.6       |
 +----------------------+------------+

succulent/api.py

@@ -12,13 +12,13 @@ class SucculentAPI:
         host (str): The host address to run the API server.
         port (int): The port number to run the API server.
         config (str): Path to the configuration file.
-        format (str, optional): The format of the data ('csv', 'json', 'sqlite', or 'image').
+        format (str, optional): The format of the data ('csv', 'json', 'sqlite', 'xml', or 'image').
 
     Attributes:
         host (str): The host address to run the API server.
         port (int): The port number to run the API server.
         config (dict): Configuration file.
-        format (str): The format of the data ('csv', 'json', 'sqlite', or 'image').
+        format (str): The format of the data ('csv', 'json', 'sqlite', 'xml', or 'image').
         app (Flask): Flask application.
         processing (Processing): Instance of the Processing class.
 

succulent/processing.py

@@ -1,4 +1,5 @@
 import os
+import re
 import hmac
 import sqlite3
 import inspect
@@ -59,6 +60,13 @@ def __init__(self, format, config, unittest=False):
             if 'password' in self.columns:
                 raise ValueError(
                     '\"password\" is a reserved keyword and cannot be used as a column name.')
+            if format == 'xml':
+                for column in self.columns:
+                    if not self.valid_xml_name(column):
+                        raise ValueError(
+                            f'"{column}" cannot be used as a column name with the XML format '
+                            '(use letters, digits, "_", "-" or "."; start with a letter or "_"; '
+                            'no spaces; must not start with "xml").')
             self.boundaries = [{
                 configuration['name']: {
                     key: configuration[key]
@@ -83,6 +91,20 @@ def __init__(self, format, config, unittest=False):
         if format == 'image':
             self.key = config['data'][0]['key']
 
+    @staticmethod
+    def valid_xml_name(name):
+        """Check whether a column name can be used as an XML element name.
+
+        Args:
+            name (str): The column name.
+
+        Returns:
+            bool: True if the name is a valid XML element name.
+        """
+        return (isinstance(name, str)
+                and re.fullmatch(r'[A-Za-z_][\w.\-]*', name, flags=re.ASCII) is not None
+                and not name.lower().startswith('xml'))
+
     def parameters(self):
         """Generate URL parameters based on data columns.
 
@@ -127,14 +149,65 @@ def boundary(self, value, boundary, column):
                 f'{column} ({value}) is greater than the specified maximum ({boundary["max"]}).')
         return value
 
+    def xml_values(self, req):
+        """Parse an XML request body into a dictionary.
+
+        Each direct child of the root element is one field, e.g.
+        ``<data><temperature>21.5</temperature></data>``. The name of the
+        root element is not checked.
+
+        Args:
+            req (Request): The incoming request object.
+
+        Raises:
+            ValueError: The body is not well-formed XML.
+
+        Returns:
+            dict: Field names mapped to their text values.
+        """
+        try:
+            root = ET.fromstring(req.data)
+        except ET.ParseError as err:
+            raise ValueError(f'Invalid XML: {err}.') from None
+
+        return {child.tag: child.text for child in root}
+
+    def request_values(self, req):
+        """Extract the submitted values from the request.
+
+        Values are read from a JSON object body, from an XML body (for the
+        'xml' format), or otherwise from the query parameters. An XML-format
+        request without a body (e.g. ``POST /measure?temperature=21`` from
+        an ESP32) is read from the query parameters, as for other formats.
+
+        Args:
+            req (Request): The incoming request object.
+
+        Raises:
+            ValueError: The body is not a JSON object, or not well-formed XML.
+
+        Returns:
+            dict: Field names mapped to their values.
+        """
+        if req.is_json:
+            if not isinstance(req.json, dict):
+                raise ValueError('Invalid JSON: the request body must be a JSON object.')
+            return req.json
+
+        if self.format == 'xml' and req.data:
+            return self.xml_values(req)
+
+        return {column: req.args.get(column, default=None) for column in self.columns}
+
     def request_passwords(self, req):
         """Collect all passwords supplied with the request.
 
         A password can be sent in any of the following places:
 
         - ``Authorization: Bearer <password>`` header,
         - ``X-API-Key: <password>`` header,
-        - ``password`` field in a JSON body,
+        - ``password`` field in a JSON body or ``<password>`` element in an
+          XML body,
         - ``password`` query parameter (kept for backward compatibility;
           prefer a header, since URLs are often written to server logs).
 
@@ -158,8 +231,16 @@ def request_passwords(self, req):
         if api_key:
             passwords.append(api_key)
 
-        if req.is_json and isinstance(req.json, dict) and req.json.get('password') is not None:
-            passwords.append(req.json['password'])
+        if req.is_json:
+            if isinstance(req.json, dict) and req.json.get('password') is not None:
+                passwords.append(req.json['password'])
+        elif self.format == 'xml' and req.data:
+            try:
+                xml_password = self.xml_values(req).get('password')
+            except ValueError:
+                xml_password = None
+            if xml_password is not None:
+                passwords.append(xml_password)
 
         query_password = req.args.get('password', default=None)
         if query_password is not None:
@@ -242,18 +323,12 @@ def process(self, req):
             else:
                 self.df = pd.DataFrame(columns=self.columns)
 
-            # Parse data from request
+            # Parse data from request (once, before checking the columns)
+            values = self.request_values(req)
+
             data = {}
             for column in self.columns:
-                # Request type
-                if req.is_json:
-                    value = req.json[column] if column in req.json else None
-                elif self.format == 'xml':
-                    root = ET.fromstring(req.data)
-                    data = {child.tag: child.text for child in root}
-                    value = data[column] if column in data else None
-                else:
-                    value = req.args.get(column, default=None)
+                value = values.get(column)
 
                 # Boundary check
                 if column in [list(boundaries.keys())[0] for boundaries in self.boundaries]:

tests/test_succulent.py

@@ -485,6 +485,91 @@ def test_data_and_export(self):
                          ['temperature,humidity', '21.5,40', '22.0,45'])
 
 
+class TestXmlRequests(unittest.TestCase):
+    """
+    Test case for submitting data with the XML format.
+    """
+
+    def setUp(self):
+        config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'password.yml')
+        self.api = SucculentAPI(host='0.0.0.0', port=8080, config=config_path, format='xml')
+        self.client = self.api.app.test_client()
+        self.xml = {'Content-Type': 'application/xml'}
+        self.key = {'X-API-Key': 's3cret'}
+
+    def tearDown(self):
+        shutil.rmtree(self.api.processing.directory, ignore_errors=True)
+
+    def last_row(self):
+        return self.api.processing.df.iloc[-1].to_dict()
+
+    def test_malformed_xml_returns_400(self):
+        for body in ['<data><temperature>21', 'hello', '<data></wrong>']:
+            with self.subTest(body=body):
+                response = self.client.post('/measure', data=body,
+                                            headers={**self.xml, **self.key})
+                self.assertEqual(response.status_code, 400)
+                self.assertTrue(response.json['message'].startswith('Invalid XML:'))
+
+    def test_query_parameters_without_body(self):
+        # ESP32-style request: no body, values in the query string
+        response = self.client.post('/measure?temperature=20&humidity=40',
+                                    headers=self.key)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(self.last_row(), {'temperature': '20', 'humidity': '40'})
+
+    def test_xml_body_without_content_type(self):
+        response = self.client.post(
+            '/measure', data='<data><temperature>24</temperature></data>', headers=self.key)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(self.last_row()['temperature'], '24')
+
+    def test_password_element_in_xml_body(self):
+        response = self.client.post(
+            '/measure', headers=self.xml,
+            data='<data><temperature>22</temperature><password>s3cret</password></data>')
+        self.assertEqual(response.status_code, 200)
+        self.assertNotIn('password', self.api.processing.df.columns)
+
+        response = self.client.post(
+            '/measure', headers=self.xml,
+            data='<data><temperature>22</temperature><password>wrong</password></data>')
+        self.assertEqual(response.status_code, 400)
+        self.assertEqual(response.json['message'], 'Unauthorised access.')
+
+    def test_json_body_with_xml_format(self):
+        response = self.client.post('/measure', json={'temperature': 23}, headers=self.key)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(self.last_row()['temperature'], 23)
+
+    def test_invalid_column_names(self):
+        for name in ['my temp', '1st', 'xmlvalue', 'a/b']:
+            with self.subTest(name=name):
+                with self.assertRaises(ValueError):
+                    Processing(config={'data': [{'name': name}]}, format='xml', unittest=True)
+
+        # Valid names, and names that are only invalid for XML in other formats
+        Processing(config={'data': [{'name': n} for n in ['temp.c', 'hum-1', '_x']]},
+                   format='xml', unittest=True)
+        Processing(config={'data': [{'name': 'my temp'}]}, format='csv', unittest=True)
+
+
+class TestJsonRequests(unittest.TestCase):
+    """
+    Test case for JSON request bodies that are not objects.
+    """
+
+    def test_non_object_json_returns_400(self):
+        config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configuration.yml')
+        api = SucculentAPI(host='0.0.0.0', port=8080, config=config_path, format='csv')
+        for body in [[1, 2], 'text', 5]:
+            with self.subTest(body=body):
+                response = api.app.test_client().post('/measure', json=body)
+                self.assertEqual(response.status_code, 400)
+                self.assertEqual(response.json['message'],
+                                 'Invalid JSON: the request body must be a JSON object.')
+
+
 class TestStaleData(unittest.TestCase):
     """
     Test that /data and /export do not return stale in-memory data.