Set the client-side cache headers, with an ETag that is the same in every process and none after errors

device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py

@@ -79,8 +79,8 @@
 from flask.helpers import make_response
 from fiftyone_devicedetection_cloud.devicedetection_cloud_pipelinebuilder import DeviceDetectionCloudPipelineBuilder
 from fiftyone_pipeline_core.logger import Logger
-from fiftyone_pipeline_core.web import webevidence, set_response_header
-import json
+from fiftyone_pipeline_core.web import (
+    javascript_response, json_response, webevidence, set_response_header)
 import os
 import sys
 
@@ -123,57 +123,28 @@ def run(self):
         GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5000)))
 
     # First we make a JSON route that will be called from the client side and will return
-    # a JSON encoded property database using any additional evidence provided by the client 
+    # a JSON encoded property database using any additional evidence provided by the client.
+    # json_response processes the request and sets the cache headers, so a browser keeps a
+    # good answer for 30 minutes and never keeps one made while processing failed.
 
     @staticmethod
     @app.route('/json', methods=['POST'])
     def jsonroute():
 
-        # Create the flowdata object for the JSON route
-        flowdata = GettingStartedWeb.pipeline.create_flowdata()
-
-        # Add any information from the request (headers, cookies and additional 
-        # client side provided information)
-
-        flowdata.evidence.add_from_dict(webevidence(request))
-
-        # Process the flowdata
-
-        flowdata.process()
-
-        # Return the JSON from the JSONBundler engine
-
-        return json.dumps(flowdata.jsonbundler.json)
+        return json_response(GettingStartedWeb.pipeline, request)
 
     # Next we serve the client-side JavaScript from its own route. The other
     # Pipeline APIs ship a web integration that intercepts '/51Degrees.core.js'
     # for this, so the page can reference the script by that name. There is no
     # such integration for Flask, so the example wires up the route itself.
+    # Query parameters are evidence, so the per-request 'fod-js-enable-cookies'
+    # override is picked up and applied by the JavaScriptBuilder engine.
 
     @staticmethod
     @app.route('/51Degrees.core.js')
     def core_js():
 
-        # Create the flowdata object for the JavaScript route
-        flowdata = GettingStartedWeb.pipeline.create_flowdata()
-
-        # Add any information from the request (headers, cookies and additional
-        # client side provided information). Query parameters are included, so
-        # the per-request 'fod-js-enable-cookies' override is picked up here and
-        # applied by the JavaScriptBuilder engine.
-
-        flowdata.evidence.add_from_dict(webevidence(request))
-
-        # Process the flowdata
-
-        flowdata.process()
-
-        # Return the JavaScript from the JavaScriptBuilder engine
-
-        response = make_response(flowdata.javascriptbuilder.javascript)
-        response.headers["Content-Type"] = "application/x-javascript"
-
-        return response
+        return javascript_response(GettingStartedWeb.pipeline, request)
 
     # In the main route we dynamically update the screen's device property display
     # using the above JSON route

device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py

@@ -82,7 +82,8 @@
 from flask import Flask, request, render_template
 from fiftyone_pipeline_core.logger import Logger
 from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder
-from fiftyone_pipeline_core.web import webevidence, set_response_header
+from fiftyone_pipeline_core.web import (
+    javascript_response, json_response, webevidence, set_response_header)
 import json
 
 class GettingStartedWeb():
@@ -108,60 +109,28 @@ def run(self):
             GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5001)))
 
     # First we make a JSON route that will be called from the client side and will return
-    # a JSON encoded property database using any additional evidence provided by the client 
+    # a JSON encoded property database using any additional evidence provided by the client.
+    # json_response processes the request and sets the cache headers, so a browser keeps a
+    # good answer for 30 minutes and never keeps one made while processing failed.
 
     @staticmethod
     @app.route('/json', methods=['POST'])
     def jsonroute():
 
-        # Create the flowdata object for the JSON route. It is closed when the
-        # route returns, which releases the results of the detection straight
-        # away rather than when the garbage collector next runs.
-        with GettingStartedWeb.pipeline.create_flowdata() as flowdata:
-
-            # Add any information from the request (headers, cookies and additional 
-            # client side provided information)
-
-            flowdata.evidence.add_from_dict(webevidence(request))
-
-            # Process the flowdata
-
-            flowdata.process()
-
-            # Return the JSON from the JSONBundler engine
-
-            return json.dumps(flowdata.jsonbundler.json)
+        return json_response(GettingStartedWeb.pipeline, request)
 
     # Next we serve the client-side JavaScript from its own route. The other
     # Pipeline APIs ship a web integration that intercepts '/51Degrees.core.js'
     # for this, so the page can reference the script by that name. There is no
     # such integration for Flask, so the example wires up the route itself.
+    # Query parameters are evidence, so the per-request 'fod-js-enable-cookies'
+    # override is picked up and applied by the JavaScriptBuilder engine.
 
     @staticmethod
     @app.route('/51Degrees.core.js')
     def core_js():
 
-        # Create the flowdata object for the JavaScript route. It is closed
-        # when the route returns.
-        with GettingStartedWeb.pipeline.create_flowdata() as flowdata:
-
-            # Add any information from the request (headers, cookies and additional
-            # client side provided information). Query parameters are included, so
-            # the per-request 'fod-js-enable-cookies' override is picked up here and
-            # applied by the JavaScriptBuilder engine.
-
-            flowdata.evidence.add_from_dict(webevidence(request))
-
-            # Process the flowdata
-
-            flowdata.process()
-
-            # Return the JavaScript from the JavaScriptBuilder engine
-
-            response = make_response(flowdata.javascriptbuilder.javascript)
-            response.headers["Content-Type"] = "application/x-javascript"
-
-            return response
+        return javascript_response(GettingStartedWeb.pipeline, request)
 
     # In the main route we dynamically update the screen's device property display
     # using the above JSON route

location/examples/cloud/web.py

@@ -32,8 +32,7 @@
 # This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/location/examples/cloud/web.py)
 
 from fiftyone_location.location_pipelinebuilder import LocationPipelineBuilder
-from fiftyone_pipeline_core.web import webevidence
-import json
+from fiftyone_pipeline_core.web import json_response, webevidence
 
 # First create the location pipeline with the desired settings.
 
@@ -68,24 +67,14 @@
     app = Flask(__name__)
 
     # First we make a JSON route that will be called from the client side and will return
-    # a JSON encoded property database using any additional evidence provided by the client
+    # a JSON encoded property database using any additional evidence provided by the client.
+    # json_response processes the request and sets the cache headers, so a browser keeps a
+    # good answer for 30 minutes and never keeps one made while processing failed.
 
     @app.route('/json', methods=['POST'])
     def jsonroute():
 
-        # Create the flowData object for the JSON route
-        flowData = pipeline.create_flowdata()
-
-        # Add any information from the request (headers, cookies and additional
-        # client side provided information)
-
-        flowData.evidence.add_from_dict(webevidence(request))
-
-        # Process the flowData
-        flowData.process()
-
-        # Return the JSON from the JSONBundler engine
-        return json.dumps(flowData.jsonbundler.json)
+        return json_response(pipeline, request)
 
     # Helper function to get a property value if it exists and return
     # the reason why if it doesn't

pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py

@@ -74,7 +74,6 @@
 # - [fiftyone_pipeline_cloudrequestengine](https://pypi.org/project/fiftyone-pipeline-cloudrequestengine/)
 # - [flask](https://pypi.org/project/flask/)
 
-import json
 import os
 import sys
 
@@ -84,7 +83,8 @@
 from fiftyone_pipeline_cloudrequestengine.cloudrequestexception import (
     CloudRequestException)
 from fiftyone_pipeline_core.logger import Logger
-from fiftyone_pipeline_core.web import set_response_header, webevidence
+from fiftyone_pipeline_core.web import (
+    javascript_response, json_response, set_response_header, webevidence)
 
 from fiftyone_pipeline_examples.cloud.engines import (
     CountriesTranslationCloud, DeviceDetectionCloud, IpIntelligenceCloud)
@@ -265,27 +265,20 @@ def index():
     # The other Pipeline APIs ship web integrations that serve the
     # client-side script at '/51Degrees.core.js', so the page refers to it by
     # that name. There is no such integration for Flask, so the example adds
-    # the route itself.
+    # the route itself. javascript_response and json_response process the
+    # request and set the cache headers, so a browser keeps a good answer
+    # for 30 minutes and never keeps one made while processing failed.
     @app.route("/51Degrees.core.js")
     def core_js():
 
-        flowdata = process()
-
-        response = make_response(flowdata.javascriptbuilder.javascript)
-        response.headers["Content-Type"] = "application/x-javascript"
-
-        return response
+        return javascript_response(pipeline, request)
 
     # The client-side script posts the evidence it gathers here, and gets
     # back the results of processing it as JSON.
     @app.route("/json", methods=["POST"])
     def json_results():
 
-        flowdata = process()
-
-        return app.response_class(
-            json.dumps(flowdata.jsonbundler.json),
-            mimetype="application/json")
+        return json_response(pipeline, request)
 
     return app
 

pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_offline.py

@@ -27,6 +27,9 @@
 """
 
 import json
+import os
+import subprocess
+import sys
 
 import pytest
 import requests
@@ -113,6 +116,9 @@ class StubCloud:
     def __init__(self, sections):
         self.sections = sections
         self.requests = []
+        # While set, a request for results fails as it does when the service
+        # cannot be reached.
+        self.failing = False
         # The Origin header of each request, kept apart from the body.
         self.origins = []
 
@@ -131,6 +137,10 @@ def request(self, method, url, data=None, headers=None):
         if "evidencekeys" in url:
             return StubResponse(["header.user-agent", "query.client-ip"])
 
+        if self.failing:
+            raise requests.exceptions.ConnectionError(
+                "The stand-in service cannot be reached.")
+
         # A request that names the properties it wants gets those alone,
         # and no section it named nothing from.
         named = data.get("values")
@@ -302,3 +312,97 @@ def test_licence_key_gets_an_engine_only_for_what_it_asks_for(cloud):
 def test_licence_key_never_reaches_the_page(cloud):
     cloud(EVERY_SECTION)
     assert "a-licence-key" not in page_for(license_key="a-licence-key")
+
+
+# The cache headers of the client-side script and the client-side callback,
+# as the cache header detail of the web integration specification sets them.
+DESKTOP_CHROME = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+CACHEABLE = "max-age=1800, private"
+NOT_CACHEABLE = "private, no-cache"
+
+
+def client_side(client, path, **headers):
+    """The response of one of the client-side endpoints."""
+
+    headers.setdefault("User-Agent", DESKTOP_CHROME)
+    if path == "/json":
+        return client.post(path, headers=headers)
+    return client.get(path, headers=headers)
+
+
+def example_client():
+    pipeline = create_pipeline(
+        Logger(min_level="error"), resource_key="a-resource-key")
+    return create_app(pipeline).test_client()
+
+
+def script_etag():
+    """The ETag of the client-side script for a fixed request, with the
+    stand-in answering. Run in a process of its own by the test that
+    compares processes, because it replaces requests.request for good."""
+
+    requests.request = StubCloud(EVERY_SECTION).request
+    response = client_side(example_client(), "/51Degrees.core.js")
+    return response.headers.get("ETag")
+
+
+@pytest.mark.parametrize("path", ["/51Degrees.core.js", "/json"])
+def test_a_good_response_may_be_kept_by_the_browser_alone(cloud, path):
+    cloud(EVERY_SECTION)
+    response = client_side(example_client(), path)
+
+    assert response.status_code == 200
+    assert response.headers.get("Cache-Control") == CACHEABLE
+    assert response.headers.get("ETag", "").startswith('"')
+    assert "User-Agent" in response.headers.get("Vary", "")
+
+
+def test_the_etag_is_the_same_in_every_process():
+    """Python seeds the hash of a string afresh in each process, so the two
+    processes are given different seeds."""
+
+    tests = os.path.dirname(os.path.abspath(__file__))
+    code = ("import sys; sys.path.insert(0, sys.argv[1]); "
+            "from test_cloud_mixed_web_offline import script_etag; "
+            "print(script_etag())")
+    etags = []
+    for seed in ("1", "2"):
+        finished = subprocess.run(
+            [sys.executable, "-c", code, tests], capture_output=True,
+            text=True, timeout=120, env=dict(os.environ, PYTHONHASHSEED=seed))
+        assert finished.returncode == 0, finished.stderr
+        etags.append(finished.stdout.strip().splitlines()[-1])
+    assert etags[0].startswith('"'), etags
+    assert etags[0] == etags[1]
+
+
+@pytest.mark.parametrize("path", ["/51Degrees.core.js", "/json"])
+def test_a_response_to_a_failed_request_carries_no_etag(cloud, path):
+    stub = cloud(EVERY_SECTION)
+    client = example_client()
+    stub.failing = True
+    response = client_side(client, path)
+
+    assert response.status_code == 200
+    assert response.headers.get("Cache-Control") == NOT_CACHEABLE
+    assert "ETag" not in response.headers
+
+
+# A 304 for the script, which is a GET, and a 412 for the callback, which is
+# a POST, as RFC 9110 section 13.1.2 asks.
+@pytest.mark.parametrize("path, status", [
+    ("/51Degrees.core.js", 304), ("/json", 412)])
+def test_a_matching_etag_is_answered_while_processing_fails(
+        cloud, path, status):
+    stub = cloud(EVERY_SECTION)
+    client = example_client()
+    etag = client_side(client, path).headers.get("ETag")
+    assert etag, "a good response carries an ETag"
+    stub.failing = True
+    response = client_side(client, path, **{"If-None-Match": etag})
+
+    assert response.status_code == status
+    if status == 304:
+        assert response.headers.get("ETag") == etag