fix: return null instead of NaN in /data JSON

succulent/api.py

@@ -1,3 +1,5 @@
+import numpy as np
+import pandas as pd
 from flask import Flask, jsonify, request, url_for, make_response
 from succulent.configuration import Configuration
 from succulent.processing import Processing
@@ -131,7 +133,12 @@ def data(self):
         if data['valid'] == False:
             return jsonify({'message': data['message']}), data.get('code', 400)
 
-        return jsonify({'data': data['data'].to_dict(orient='records')}), 200
+        # Missing values (NaN) and infinities are not valid JSON, so send them
+        # as null, which every JSON client (e.g. JavaScript) can read
+        df = data['data'].replace([np.inf, -np.inf], np.nan)
+        df = df.astype(object).where(pd.notna(df), None)
+
+        return jsonify({'data': df.to_dict(orient='records')}), 200
 
     def export(self):
         """Export the stored data to a CSV file.

tests/test_succulent.py

@@ -1,6 +1,7 @@
 from succulent import __version__
 import pandas as pd
 import unittest
+import json
 import os
 import shutil
 import pytest
@@ -509,6 +510,57 @@ def test_removed_data_file_is_not_served(self):
                 self.assertEqual(response.json['message'], 'No data available.')
 
 
+class TestDataJson(unittest.TestCase):
+    """
+    Test that /data always returns valid (strict) JSON.
+    """
+
+    def strict_json(self, raw):
+        # Like JavaScript's JSON.parse: reject NaN, Infinity and -Infinity
+        def reject(constant):
+            raise ValueError(f'Invalid JSON constant: {constant}')
+        return json.loads(raw, parse_constant=reject)
+
+    def test_missing_value_is_null(self):
+        config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'password.yml')
+        headers = {'X-API-Key': 's3cret'}
+
+        for fmt in ['csv', 'json', 'sqlite', 'xml']:
+            with self.subTest(format=fmt):
+                api = SucculentAPI(host='0.0.0.0', port=8080, config=config_path, format=fmt)
+                client = api.app.test_client()
+
+                # 'humidity' is not sent, so it is stored as missing
+                if fmt == 'xml':
+                    client.post('/measure', data='<data><temperature>21</temperature></data>',
+                                headers={**headers, 'Content-Type': 'application/xml'})
+                else:
+                    client.post('/measure', json={'temperature': 21}, headers=headers)
+
+                response = client.get('/data', headers=headers)
+                self.assertEqual(response.status_code, 200)
+
+                data = self.strict_json(response.data.decode())['data']
+                self.assertEqual(len(data), 1)
+                self.assertIsNone(data[0]['humidity'])
+                self.assertEqual(float(data[0]['temperature']), 21.0)
+
+                shutil.rmtree(api.processing.directory, ignore_errors=True)
+
+    def test_infinity_is_null(self):
+        config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'password.yml')
+        headers = {'X-API-Key': 's3cret'}
+        api = SucculentAPI(host='0.0.0.0', port=8080, config=config_path, format='csv')
+        client = api.app.test_client()
+
+        # 'humidity' has no boundary, so 'inf' is accepted and read back as a float
+        client.post('/measure', json={'temperature': 21, 'humidity': 'inf'}, headers=headers)
+
+        response = client.get('/data', headers=headers)
+        data = self.strict_json(response.data.decode())['data']
+        self.assertIsNone(data[0]['humidity'])
+
+
 class TestConfiguration(unittest.TestCase):
     """
     Test case for the Configuration class.