diff --git a/README.md b/README.md index 69300c4..8906951 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,17 @@ timestamp: true # false by default To access the URL for data collection, send a GET request (or navigate) to [http://localhost:8080/measure](http://localhost:8080/measure). +To store data, send a POST request to `/measure` with the values in one of the following ways: + +- query parameters, e.g. `POST /measure?temperature=21.5&humidity=40` (works with every format), +- a JSON object body, e.g. `{"temperature": 21.5, "humidity": 40}` (works with every format), +- an XML body (with `format='xml'`), where each child of the root element is one value; the name of the root element is not checked: + ```sh + curl -X POST http://localhost:8080/measure -H "Content-Type: application/xml" -d '21.540' + ``` + +A malformed XML body or a JSON body that is not an object is rejected with a `400` response. With `format='xml'`, column names must be valid XML element names: letters, digits, `_`, `-` or `.`, starting with a letter or `_`, without spaces, and not starting with `xml`. + To restrict access to the collected data, define the following setting in the `configuration.yml` file in the root directory: ```yml password: 'password' # Password for data access @@ -220,7 +231,7 @@ The password can be sent with any request (`/measure`, `/data` and `/export`) in - `Authorization: Bearer password` header (recommended), - `X-API-Key: password` header, -- `"password": "password"` field in a JSON body (`/measure` only), +- `"password": "password"` field in a JSON body or `password` element in an XML body (`/measure` only), - `?password=password` query parameter (supported for backward compatibility). Prefer one of the headers, since URLs (including query parameters) are often written to server and proxy logs. For example: diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 4ce52e1..e4ad4d7 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -83,6 +83,18 @@ To store data collection timestamps, define the following setting in the ``confi To access the URL for data collection, send a GET request (or navigate) to `http://localhost:8080/measure `_. +To store data, send a POST request to ``/measure`` with the values in one of the following ways: + +- query parameters, e.g. ``POST /measure?temperature=21.5&humidity=40`` (works with every format), +- a JSON object body, e.g. ``{"temperature": 21.5, "humidity": 40}`` (works with every format), +- an XML body (with ``format='xml'``), where each child of the root element is one value; the name of the root element is not checked: + +.. code:: sh + + curl -X POST http://localhost:8080/measure -H "Content-Type: application/xml" -d '21.540' + +A malformed XML body or a JSON body that is not an object is rejected with a ``400`` response. With ``format='xml'``, column names must be valid XML element names: letters, digits, ``_``, ``-`` or ``.``, starting with a letter or ``_``, without spaces, and not starting with ``xml``. + To restrict access to the collected data, define the following setting in the ``configuration.yml`` file in the root directory: .. code:: yaml @@ -93,7 +105,7 @@ The password can be sent with any request (``/measure``, ``/data`` and ``/export - ``Authorization: Bearer password`` header (recommended), - ``X-API-Key: password`` header, -- ``"password": "password"`` field in a JSON body (``/measure`` only), +- ``"password": "password"`` field in a JSON body or ``password`` element in an XML body (``/measure`` only), - ``?password=password`` query parameter (supported for backward compatibility). Prefer one of the headers, since URLs (including query parameters) are often written to server and proxy logs. For example: diff --git a/docs/installation.rst b/docs/installation.rst index 605d8ce..684641d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -26,15 +26,15 @@ List of succulent's dependencies: +----------------------+----------------------+ | Package | Version | +======================+======================+ -| pandas | ^2.0.1 | +| pandas | >=2.0.1,<3.0.0 | +----------------------+----------------------+ -| pyyaml | ^6.0 | +| pyyaml | >=6.0,<7.0 | +----------------------+----------------------+ -| flask | ^2.3.2 | +| flask | >=2.3.2,<4.0.0 | +----------------------+----------------------+ -| xmltodict | ^0.13.0 | +| lxml | >=5.2.2,<7.0.0 | +----------------------+----------------------+ -| lxml | ^5.1.0 | +| numpy | >=1.26.0,<3.0.0 | +----------------------+----------------------+ List of succulent's development dependencies: @@ -42,11 +42,11 @@ List of succulent's development dependencies: +----------------------+------------+ | Package | Version | +======================+============+ -| pytest | ^6.2 | +| pytest | ^9.0.3 | +----------------------+------------+ -| sphinx | ^4.4.0 | +| sphinx | >=7.0 | +----------------------+------------+ -| sphinx-rtd-theme | ^1.0.0 | +| sphinx-rtd-theme | >=2.0 | +----------------------+------------+ -| sphinxcontrib-bibtex | ^2.4.1 | +| sphinxcontrib-bibtex | ^2.6 | +----------------------+------------+ \ No newline at end of file diff --git a/succulent/api.py b/succulent/api.py index 2a2cc4b..f84c622 100644 --- a/succulent/api.py +++ b/succulent/api.py @@ -12,13 +12,13 @@ class SucculentAPI: host (str): The host address to run the API server. port (int): The port number to run the API server. config (str): Path to the configuration file. - format (str, optional): The format of the data ('csv', 'json', 'sqlite', or 'image'). + format (str, optional): The format of the data ('csv', 'json', 'sqlite', 'xml', or 'image'). Attributes: host (str): The host address to run the API server. port (int): The port number to run the API server. config (dict): Configuration file. - format (str): The format of the data ('csv', 'json', 'sqlite', or 'image'). + format (str): The format of the data ('csv', 'json', 'sqlite', 'xml', or 'image'). app (Flask): Flask application. processing (Processing): Instance of the Processing class. diff --git a/succulent/processing.py b/succulent/processing.py index 51486a5..840062e 100644 --- a/succulent/processing.py +++ b/succulent/processing.py @@ -1,4 +1,5 @@ import os +import re import hmac import sqlite3 import inspect @@ -59,6 +60,13 @@ def __init__(self, format, config, unittest=False): if 'password' in self.columns: raise ValueError( '\"password\" is a reserved keyword and cannot be used as a column name.') + if format == 'xml': + for column in self.columns: + if not self.valid_xml_name(column): + raise ValueError( + f'"{column}" cannot be used as a column name with the XML format ' + '(use letters, digits, "_", "-" or "."; start with a letter or "_"; ' + 'no spaces; must not start with "xml").') self.boundaries = [{ configuration['name']: { key: configuration[key] @@ -83,6 +91,20 @@ def __init__(self, format, config, unittest=False): if format == 'image': self.key = config['data'][0]['key'] + @staticmethod + def valid_xml_name(name): + """Check whether a column name can be used as an XML element name. + + Args: + name (str): The column name. + + Returns: + bool: True if the name is a valid XML element name. + """ + return (isinstance(name, str) + and re.fullmatch(r'[A-Za-z_][\w.\-]*', name, flags=re.ASCII) is not None + and not name.lower().startswith('xml')) + def parameters(self): """Generate URL parameters based on data columns. @@ -127,6 +149,56 @@ def boundary(self, value, boundary, column): f'{column} ({value}) is greater than the specified maximum ({boundary["max"]}).') return value + def xml_values(self, req): + """Parse an XML request body into a dictionary. + + Each direct child of the root element is one field, e.g. + ``21.5``. The name of the + root element is not checked. + + Args: + req (Request): The incoming request object. + + Raises: + ValueError: The body is not well-formed XML. + + Returns: + dict: Field names mapped to their text values. + """ + try: + root = ET.fromstring(req.data) + except ET.ParseError as err: + raise ValueError(f'Invalid XML: {err}.') from None + + return {child.tag: child.text for child in root} + + def request_values(self, req): + """Extract the submitted values from the request. + + Values are read from a JSON object body, from an XML body (for the + 'xml' format), or otherwise from the query parameters. An XML-format + request without a body (e.g. ``POST /measure?temperature=21`` from + an ESP32) is read from the query parameters, as for other formats. + + Args: + req (Request): The incoming request object. + + Raises: + ValueError: The body is not a JSON object, or not well-formed XML. + + Returns: + dict: Field names mapped to their values. + """ + if req.is_json: + if not isinstance(req.json, dict): + raise ValueError('Invalid JSON: the request body must be a JSON object.') + return req.json + + if self.format == 'xml' and req.data: + return self.xml_values(req) + + return {column: req.args.get(column, default=None) for column in self.columns} + def request_passwords(self, req): """Collect all passwords supplied with the request. @@ -134,7 +206,8 @@ def request_passwords(self, req): - ``Authorization: Bearer `` header, - ``X-API-Key: `` header, - - ``password`` field in a JSON body, + - ``password`` field in a JSON body or ```` element in an + XML body, - ``password`` query parameter (kept for backward compatibility; prefer a header, since URLs are often written to server logs). @@ -158,8 +231,16 @@ def request_passwords(self, req): if api_key: passwords.append(api_key) - if req.is_json and isinstance(req.json, dict) and req.json.get('password') is not None: - passwords.append(req.json['password']) + if req.is_json: + if isinstance(req.json, dict) and req.json.get('password') is not None: + passwords.append(req.json['password']) + elif self.format == 'xml' and req.data: + try: + xml_password = self.xml_values(req).get('password') + except ValueError: + xml_password = None + if xml_password is not None: + passwords.append(xml_password) query_password = req.args.get('password', default=None) if query_password is not None: @@ -242,18 +323,12 @@ def process(self, req): else: self.df = pd.DataFrame(columns=self.columns) - # Parse data from request + # Parse data from request (once, before checking the columns) + values = self.request_values(req) + data = {} for column in self.columns: - # Request type - if req.is_json: - value = req.json[column] if column in req.json else None - elif self.format == 'xml': - root = ET.fromstring(req.data) - data = {child.tag: child.text for child in root} - value = data[column] if column in data else None - else: - value = req.args.get(column, default=None) + value = values.get(column) # Boundary check if column in [list(boundaries.keys())[0] for boundaries in self.boundaries]: diff --git a/tests/test_succulent.py b/tests/test_succulent.py index fd4ae33..a53bbb1 100644 --- a/tests/test_succulent.py +++ b/tests/test_succulent.py @@ -485,6 +485,91 @@ def test_data_and_export(self): ['temperature,humidity', '21.5,40', '22.0,45']) +class TestXmlRequests(unittest.TestCase): + """ + Test case for submitting data with the XML format. + """ + + def setUp(self): + config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'password.yml') + self.api = SucculentAPI(host='0.0.0.0', port=8080, config=config_path, format='xml') + self.client = self.api.app.test_client() + self.xml = {'Content-Type': 'application/xml'} + self.key = {'X-API-Key': 's3cret'} + + def tearDown(self): + shutil.rmtree(self.api.processing.directory, ignore_errors=True) + + def last_row(self): + return self.api.processing.df.iloc[-1].to_dict() + + def test_malformed_xml_returns_400(self): + for body in ['21', 'hello', '']: + with self.subTest(body=body): + response = self.client.post('/measure', data=body, + headers={**self.xml, **self.key}) + self.assertEqual(response.status_code, 400) + self.assertTrue(response.json['message'].startswith('Invalid XML:')) + + def test_query_parameters_without_body(self): + # ESP32-style request: no body, values in the query string + response = self.client.post('/measure?temperature=20&humidity=40', + headers=self.key) + self.assertEqual(response.status_code, 200) + self.assertEqual(self.last_row(), {'temperature': '20', 'humidity': '40'}) + + def test_xml_body_without_content_type(self): + response = self.client.post( + '/measure', data='24', headers=self.key) + self.assertEqual(response.status_code, 200) + self.assertEqual(self.last_row()['temperature'], '24') + + def test_password_element_in_xml_body(self): + response = self.client.post( + '/measure', headers=self.xml, + data='22s3cret') + self.assertEqual(response.status_code, 200) + self.assertNotIn('password', self.api.processing.df.columns) + + response = self.client.post( + '/measure', headers=self.xml, + data='22wrong') + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json['message'], 'Unauthorised access.') + + def test_json_body_with_xml_format(self): + response = self.client.post('/measure', json={'temperature': 23}, headers=self.key) + self.assertEqual(response.status_code, 200) + self.assertEqual(self.last_row()['temperature'], 23) + + def test_invalid_column_names(self): + for name in ['my temp', '1st', 'xmlvalue', 'a/b']: + with self.subTest(name=name): + with self.assertRaises(ValueError): + Processing(config={'data': [{'name': name}]}, format='xml', unittest=True) + + # Valid names, and names that are only invalid for XML in other formats + Processing(config={'data': [{'name': n} for n in ['temp.c', 'hum-1', '_x']]}, + format='xml', unittest=True) + Processing(config={'data': [{'name': 'my temp'}]}, format='csv', unittest=True) + + +class TestJsonRequests(unittest.TestCase): + """ + Test case for JSON request bodies that are not objects. + """ + + def test_non_object_json_returns_400(self): + config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configuration.yml') + api = SucculentAPI(host='0.0.0.0', port=8080, config=config_path, format='csv') + for body in [[1, 2], 'text', 5]: + with self.subTest(body=body): + response = api.app.test_client().post('/measure', json=body) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json['message'], + 'Invalid JSON: the request body must be a JSON object.') + + class TestStaleData(unittest.TestCase): """ Test that /data and /export do not return stale in-memory data.