diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py index 8260065b..93f448b3 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py @@ -79,8 +79,8 @@ from flask.helpers import make_response from fiftyone_devicedetection_cloud.devicedetection_cloud_pipelinebuilder import DeviceDetectionCloudPipelineBuilder from fiftyone_pipeline_core.logger import Logger -from fiftyone_pipeline_core.web import webevidence, set_response_header -import json +from fiftyone_pipeline_core.web import ( + javascript_response, json_response, webevidence, set_response_header) import os import sys @@ -123,57 +123,28 @@ def run(self): GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5000))) # First we make a JSON route that will be called from the client side and will return - # a JSON encoded property database using any additional evidence provided by the client + # a JSON encoded property database using any additional evidence provided by the client. + # json_response processes the request and sets the cache headers, so a browser keeps a + # good answer for 30 minutes and never keeps one made while processing failed. @staticmethod @app.route('/json', methods=['POST']) def jsonroute(): - # Create the flowdata object for the JSON route - flowdata = GettingStartedWeb.pipeline.create_flowdata() - - # Add any information from the request (headers, cookies and additional - # client side provided information) - - flowdata.evidence.add_from_dict(webevidence(request)) - - # Process the flowdata - - flowdata.process() - - # Return the JSON from the JSONBundler engine - - return json.dumps(flowdata.jsonbundler.json) + return json_response(GettingStartedWeb.pipeline, request) # Next we serve the client-side JavaScript from its own route. The other # Pipeline APIs ship a web integration that intercepts '/51Degrees.core.js' # for this, so the page can reference the script by that name. There is no # such integration for Flask, so the example wires up the route itself. + # Query parameters are evidence, so the per-request 'fod-js-enable-cookies' + # override is picked up and applied by the JavaScriptBuilder engine. @staticmethod @app.route('/51Degrees.core.js') def core_js(): - # Create the flowdata object for the JavaScript route - flowdata = GettingStartedWeb.pipeline.create_flowdata() - - # Add any information from the request (headers, cookies and additional - # client side provided information). Query parameters are included, so - # the per-request 'fod-js-enable-cookies' override is picked up here and - # applied by the JavaScriptBuilder engine. - - flowdata.evidence.add_from_dict(webevidence(request)) - - # Process the flowdata - - flowdata.process() - - # Return the JavaScript from the JavaScriptBuilder engine - - response = make_response(flowdata.javascriptbuilder.javascript) - response.headers["Content-Type"] = "application/x-javascript" - - return response + return javascript_response(GettingStartedWeb.pipeline, request) # In the main route we dynamically update the screen's device property display # using the above JSON route diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py index a8e1e3ce..e7242044 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py @@ -82,7 +82,8 @@ from flask import Flask, request, render_template from fiftyone_pipeline_core.logger import Logger from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder -from fiftyone_pipeline_core.web import webevidence, set_response_header +from fiftyone_pipeline_core.web import ( + javascript_response, json_response, webevidence, set_response_header) import json class GettingStartedWeb(): @@ -108,60 +109,28 @@ def run(self): GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5001))) # First we make a JSON route that will be called from the client side and will return - # a JSON encoded property database using any additional evidence provided by the client + # a JSON encoded property database using any additional evidence provided by the client. + # json_response processes the request and sets the cache headers, so a browser keeps a + # good answer for 30 minutes and never keeps one made while processing failed. @staticmethod @app.route('/json', methods=['POST']) def jsonroute(): - # Create the flowdata object for the JSON route. It is closed when the - # route returns, which releases the results of the detection straight - # away rather than when the garbage collector next runs. - with GettingStartedWeb.pipeline.create_flowdata() as flowdata: - - # Add any information from the request (headers, cookies and additional - # client side provided information) - - flowdata.evidence.add_from_dict(webevidence(request)) - - # Process the flowdata - - flowdata.process() - - # Return the JSON from the JSONBundler engine - - return json.dumps(flowdata.jsonbundler.json) + return json_response(GettingStartedWeb.pipeline, request) # Next we serve the client-side JavaScript from its own route. The other # Pipeline APIs ship a web integration that intercepts '/51Degrees.core.js' # for this, so the page can reference the script by that name. There is no # such integration for Flask, so the example wires up the route itself. + # Query parameters are evidence, so the per-request 'fod-js-enable-cookies' + # override is picked up and applied by the JavaScriptBuilder engine. @staticmethod @app.route('/51Degrees.core.js') def core_js(): - # Create the flowdata object for the JavaScript route. It is closed - # when the route returns. - with GettingStartedWeb.pipeline.create_flowdata() as flowdata: - - # Add any information from the request (headers, cookies and additional - # client side provided information). Query parameters are included, so - # the per-request 'fod-js-enable-cookies' override is picked up here and - # applied by the JavaScriptBuilder engine. - - flowdata.evidence.add_from_dict(webevidence(request)) - - # Process the flowdata - - flowdata.process() - - # Return the JavaScript from the JavaScriptBuilder engine - - response = make_response(flowdata.javascriptbuilder.javascript) - response.headers["Content-Type"] = "application/x-javascript" - - return response + return javascript_response(GettingStartedWeb.pipeline, request) # In the main route we dynamically update the screen's device property display # using the above JSON route diff --git a/location/examples/cloud/web.py b/location/examples/cloud/web.py index 93c8e3aa..f0ea5a45 100644 --- a/location/examples/cloud/web.py +++ b/location/examples/cloud/web.py @@ -32,8 +32,7 @@ # This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/location/examples/cloud/web.py) from fiftyone_location.location_pipelinebuilder import LocationPipelineBuilder -from fiftyone_pipeline_core.web import webevidence -import json +from fiftyone_pipeline_core.web import json_response, webevidence # First create the location pipeline with the desired settings. @@ -68,24 +67,14 @@ app = Flask(__name__) # First we make a JSON route that will be called from the client side and will return - # a JSON encoded property database using any additional evidence provided by the client + # a JSON encoded property database using any additional evidence provided by the client. + # json_response processes the request and sets the cache headers, so a browser keeps a + # good answer for 30 minutes and never keeps one made while processing failed. @app.route('/json', methods=['POST']) def jsonroute(): - # Create the flowData object for the JSON route - flowData = pipeline.create_flowdata() - - # Add any information from the request (headers, cookies and additional - # client side provided information) - - flowData.evidence.add_from_dict(webevidence(request)) - - # Process the flowData - flowData.process() - - # Return the JSON from the JSONBundler engine - return json.dumps(flowData.jsonbundler.json) + return json_response(pipeline, request) # Helper function to get a property value if it exists and return # the reason why if it doesn't diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py index 66856b86..f4569463 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py @@ -74,7 +74,6 @@ # - [fiftyone_pipeline_cloudrequestengine](https://pypi.org/project/fiftyone-pipeline-cloudrequestengine/) # - [flask](https://pypi.org/project/flask/) -import json import os import sys @@ -84,7 +83,8 @@ from fiftyone_pipeline_cloudrequestengine.cloudrequestexception import ( CloudRequestException) from fiftyone_pipeline_core.logger import Logger -from fiftyone_pipeline_core.web import set_response_header, webevidence +from fiftyone_pipeline_core.web import ( + javascript_response, json_response, set_response_header, webevidence) from fiftyone_pipeline_examples.cloud.engines import ( CountriesTranslationCloud, DeviceDetectionCloud, IpIntelligenceCloud) @@ -265,27 +265,20 @@ def index(): # The other Pipeline APIs ship web integrations that serve the # client-side script at '/51Degrees.core.js', so the page refers to it by # that name. There is no such integration for Flask, so the example adds - # the route itself. + # the route itself. javascript_response and json_response process the + # request and set the cache headers, so a browser keeps a good answer + # for 30 minutes and never keeps one made while processing failed. @app.route("/51Degrees.core.js") def core_js(): - flowdata = process() - - response = make_response(flowdata.javascriptbuilder.javascript) - response.headers["Content-Type"] = "application/x-javascript" - - return response + return javascript_response(pipeline, request) # The client-side script posts the evidence it gathers here, and gets # back the results of processing it as JSON. @app.route("/json", methods=["POST"]) def json_results(): - flowdata = process() - - return app.response_class( - json.dumps(flowdata.jsonbundler.json), - mimetype="application/json") + return json_response(pipeline, request) return app diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_offline.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_offline.py index 47fe209e..d2ea3516 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_offline.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_offline.py @@ -27,6 +27,9 @@ """ import json +import os +import subprocess +import sys import pytest import requests @@ -113,6 +116,9 @@ class StubCloud: def __init__(self, sections): self.sections = sections self.requests = [] + # While set, a request for results fails as it does when the service + # cannot be reached. + self.failing = False # The Origin header of each request, kept apart from the body. self.origins = [] @@ -131,6 +137,10 @@ def request(self, method, url, data=None, headers=None): if "evidencekeys" in url: return StubResponse(["header.user-agent", "query.client-ip"]) + if self.failing: + raise requests.exceptions.ConnectionError( + "The stand-in service cannot be reached.") + # A request that names the properties it wants gets those alone, # and no section it named nothing from. named = data.get("values") @@ -302,3 +312,97 @@ def test_licence_key_gets_an_engine_only_for_what_it_asks_for(cloud): def test_licence_key_never_reaches_the_page(cloud): cloud(EVERY_SECTION) assert "a-licence-key" not in page_for(license_key="a-licence-key") + + +# The cache headers of the client-side script and the client-side callback, +# as the cache header detail of the web integration specification sets them. +DESKTOP_CHROME = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") +CACHEABLE = "max-age=1800, private" +NOT_CACHEABLE = "private, no-cache" + + +def client_side(client, path, **headers): + """The response of one of the client-side endpoints.""" + + headers.setdefault("User-Agent", DESKTOP_CHROME) + if path == "/json": + return client.post(path, headers=headers) + return client.get(path, headers=headers) + + +def example_client(): + pipeline = create_pipeline( + Logger(min_level="error"), resource_key="a-resource-key") + return create_app(pipeline).test_client() + + +def script_etag(): + """The ETag of the client-side script for a fixed request, with the + stand-in answering. Run in a process of its own by the test that + compares processes, because it replaces requests.request for good.""" + + requests.request = StubCloud(EVERY_SECTION).request + response = client_side(example_client(), "/51Degrees.core.js") + return response.headers.get("ETag") + + +@pytest.mark.parametrize("path", ["/51Degrees.core.js", "/json"]) +def test_a_good_response_may_be_kept_by_the_browser_alone(cloud, path): + cloud(EVERY_SECTION) + response = client_side(example_client(), path) + + assert response.status_code == 200 + assert response.headers.get("Cache-Control") == CACHEABLE + assert response.headers.get("ETag", "").startswith('"') + assert "User-Agent" in response.headers.get("Vary", "") + + +def test_the_etag_is_the_same_in_every_process(): + """Python seeds the hash of a string afresh in each process, so the two + processes are given different seeds.""" + + tests = os.path.dirname(os.path.abspath(__file__)) + code = ("import sys; sys.path.insert(0, sys.argv[1]); " + "from test_cloud_mixed_web_offline import script_etag; " + "print(script_etag())") + etags = [] + for seed in ("1", "2"): + finished = subprocess.run( + [sys.executable, "-c", code, tests], capture_output=True, + text=True, timeout=120, env=dict(os.environ, PYTHONHASHSEED=seed)) + assert finished.returncode == 0, finished.stderr + etags.append(finished.stdout.strip().splitlines()[-1]) + assert etags[0].startswith('"'), etags + assert etags[0] == etags[1] + + +@pytest.mark.parametrize("path", ["/51Degrees.core.js", "/json"]) +def test_a_response_to_a_failed_request_carries_no_etag(cloud, path): + stub = cloud(EVERY_SECTION) + client = example_client() + stub.failing = True + response = client_side(client, path) + + assert response.status_code == 200 + assert response.headers.get("Cache-Control") == NOT_CACHEABLE + assert "ETag" not in response.headers + + +# A 304 for the script, which is a GET, and a 412 for the callback, which is +# a POST, as RFC 9110 section 13.1.2 asks. +@pytest.mark.parametrize("path, status", [ + ("/51Degrees.core.js", 304), ("/json", 412)]) +def test_a_matching_etag_is_answered_while_processing_fails( + cloud, path, status): + stub = cloud(EVERY_SECTION) + client = example_client() + etag = client_side(client, path).headers.get("ETag") + assert etag, "a good response carries an ETag" + stub.failing = True + response = client_side(client, path, **{"If-None-Match": etag}) + + assert response.status_code == status + if status == 304: + assert response.headers.get("ETag") == etag diff --git a/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py b/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py index 6edda1c2..e25c5723 100644 --- a/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py +++ b/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py @@ -31,12 +31,11 @@ # a person is in the northern or southern hemispheres. from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder -from fiftyone_pipeline_core.web import webevidence +from fiftyone_pipeline_core.web import json_response, webevidence from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary -import json # Function to get star sign from month and day def getStarSign(month, day): @@ -168,26 +167,14 @@ def get_evidence_key_filter(self): app = Flask(__name__) -# Make a route that returns back JSON encded properties for client side scripts to pick up +# Make a route that returns back JSON encoded properties for client side scripts to pick +# up. json_response processes the request and sets the cache headers, so a browser keeps +# a good answer for 30 minutes and never keeps one made while processing failed. @app.route('/json', methods=['POST']) def jsonroute(): - # Create the flowdata object for the JSON route - flowdata = myPipeline.create_flowdata() - - # Add any information from the request (headers, cookies and additional - # client side provided information) - - flowdata.evidence.add_from_dict(webevidence(request)) - - # Process the flowdata - - flowdata.process() - - # Return the JSON from the JSONBundler engine - - return json.dumps(flowdata.jsonbundler.json) + return json_response(myPipeline, request) @app.route('/') diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py index d3f96cf0..2c1d66b1 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py @@ -20,7 +20,27 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -from flask import request +import hashlib +import json + +from flask import make_response, request + +# The Cache-Control of a client-side response to a request whose processing +# recorded no errors. The browser alone may keep it, for 30 minutes. +_CACHE_CONTROL = "max-age=1800, private" + +# The Cache-Control of a client-side response to a request whose processing +# recorded errors. It goes with no ETag, so a browser that kept the response +# has nothing to revalidate with and asks again in full once processing +# works, rather than having the faulty response confirmed by a 304. +_CACHE_CONTROL_AFTER_ERRORS = "private, no-cache" + +# The headers a browser adds when it revalidates or reloads. They differ +# between a request and its revalidation and no element uses them, so they +# are left out of the ETag and the Vary header. +_REVALIDATION_HEADERS = frozenset(( + "cache-control", "if-match", "if-modified-since", "if-none-match", + "if-range", "if-unmodified-since", "pragma")) def webevidence(request): @@ -90,4 +110,146 @@ def set_response_header(flowData, response): if response_value != "": response.headers[response_key] = response_value - return response \ No newline at end of file + return response + +def javascript_response(pipeline, request): + + """! + Process a request for the client-side JavaScript and answer with the + JavaScript the pipeline's JavaScriptBuilder writes, with the cache headers + the web integration specification sets. + + A GET or HEAD request whose If-None-Match header names the ETag of its + evidence gets a 304 without being processed, so a browser keeps a good + copy while processing fails. A request by any other method whose + If-None-Match matches, such as a POST to the JSON endpoint, gets 412 + Precondition Failed, as RFC 9110 section 13.1.2 asks. Otherwise the + response carries + 'Cache-Control: max-age=1800, private' and the ETag, or + 'Cache-Control: private, no-cache' and no ETag when processing recorded + errors. Both carry a Vary header naming the request headers the + pipeline's elements accept. + + @type pipeline: Pipeline + @param pipeline: A pipeline holding the JavaScriptBuilder + @type request: Request + @param request: The request for the JavaScript + @rtype Response + @return The response to send + """ + + return _client_side_response( + pipeline, request, + lambda flowdata: flowdata.javascriptbuilder.javascript, + "application/x-javascript") + +def json_response(pipeline, request): + + """! + Process a request from the client-side JavaScript and answer with the + JSON the pipeline's JSONBundler writes, with the same cache headers as + javascript_response. + + @type pipeline: Pipeline + @param pipeline: A pipeline holding the JSONBundler + @type request: Request + @param request: The request from the client-side JavaScript + @rtype Response + @return The response to send + """ + + return _client_side_response( + pipeline, request, + lambda flowdata: json.dumps(flowdata.jsonbundler.json), + "application/json") + +def _client_side_response(pipeline, request, content, content_type): + + flowdata = pipeline.create_flowdata() + try: + evidence = webevidence(request) + flowdata.evidence.add_from_dict(evidence) + + # From the request's own evidence, because processing adds to it. + # The sequence element gives a request with no session id a new one. + etag = _etag(pipeline, evidence) + vary = _vary(pipeline, request) + + if _etag_matches(request.headers.get("If-None-Match"), etag): + if request.method not in ("GET", "HEAD"): + # RFC 9110 section 13.1.2 allows a 304 for GET and HEAD only. + response = make_response("", 412) + response.headers.pop("Content-Type", None) + return response + # With the headers of a good response that RFC 9110 section + # 15.4.5 asks a 304 to repeat. + response = make_response("", 304) + response.headers.pop("Content-Type", None) + response.headers["Cache-Control"] = _CACHE_CONTROL + response.headers["ETag"] = etag + else: + flowdata.process() + response = make_response(content(flowdata)) + response.headers["Content-Type"] = content_type + if flowdata.errors: + response.headers["Cache-Control"] = _CACHE_CONTROL_AFTER_ERRORS + else: + response.headers["Cache-Control"] = _CACHE_CONTROL + response.headers["ETag"] = etag + + if vary: + response.headers["Vary"] = vary + return response + finally: + flowdata.close() + +def _uses(pipeline, key): + + # Whether the key is evidence one of the pipeline's elements accepts. The + # web request itself, which the usage sharing element reads, is evidence + # no element uses, whatever the filters accept. + if key.lower() == "server.http-request": + return False + if key.startswith("header.") and \ + key[len("header."):] in _REVALIDATION_HEADERS: + return False + return any( + element.filter_evidence_key(key) + for element in pipeline.flow_elements) + +def _etag(pipeline, evidence): + + # SHA-256 rather than hash(), which Python seeds afresh in each process, + # so a browser revalidating against another process, after a restart or + # behind a load balancer, still matches. + digest = hashlib.sha256() + for key in sorted(evidence): + if _uses(pipeline, key): + digest.update(key.encode("utf-8")) + digest.update(b"\0") + digest.update(str(evidence[key]).encode("utf-8")) + digest.update(b"\0") + return '"' + digest.hexdigest()[:32] + '"' + +def _etag_matches(if_none_match, etag): + + # The weak comparison RFC 9110 section 13.1.2 asks for, so a W/ prefix + # is ignored, and "*" matches any ETag. + if not if_none_match: + return False + for candidate in if_none_match.split(","): + candidate = candidate.strip() + if candidate.startswith("W/"): + candidate = candidate[2:] + if candidate == "*" or candidate == etag: + return True + return False + +def _vary(pipeline, request): + + names = {} + for name, _ in request.headers: + key = name.lower() + if key not in names and _uses(pipeline, "header." + key): + names[key] = name + return ", ".join(sorted(names.values(), key=str.lower)) \ No newline at end of file diff --git a/pipeline/fiftyone_pipeline_core/tests/test_web_client_side.py b/pipeline/fiftyone_pipeline_core/tests/test_web_client_side.py new file mode 100644 index 00000000..19f81304 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_web_client_side.py @@ -0,0 +1,264 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The cache headers of the client-side JavaScript and JSON responses, as +the cache header detail of the web integration specification sets them.""" + +import os +import subprocess +import sys +from types import MappingProxyType +import unittest +from unittest import mock + +from flask import Flask, request + +from fiftyone_pipeline_core import web +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_core.web import javascript_response, json_response + +CHROME = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36") +SAFARI = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 " + "Safari/605.1.15") + +CACHEABLE = "max-age=1800, private" +NOT_CACHEABLE = "private, no-cache" + + +class Switchable(FlowElement): + """An element that fails while failing is set, as a cloud request does + while the service cannot be reached.""" + + failing = False + + def __init__(self): + super(Switchable, self).__init__() + self.datakey = "switchable" + self.properties = {"answer": {"type": "int"}} + + def process_internal(self, flowdata): + if Switchable.failing: + raise Exception("The stand-in service cannot be reached.") + flowdata.set_element_data(ElementDataDictionary(self, {"answer": 42})) + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(["header.user-agent"]) + + +def client(): + """A test client of an application serving the two client-side + endpoints.""" + + pipeline = PipelineBuilder({ + "javascript_builder_settings": {"endpoint": "/json"}, + "suppress_process_exceptions": True, + }).add(Switchable()).build() + + app = Flask(__name__) + + @app.route("/51Degrees.core.js") + def javascript(): + return javascript_response(pipeline, request) + + @app.route("/json", methods=["POST"]) + def results(): + return json_response(pipeline, request) + + return app.test_client() + + +def script_etag(): + """The ETag of the JavaScript for a fixed request. Run in a process of + its own by the test that compares processes.""" + + return client().get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}).headers["ETag"] + + +class ClientSideCacheTests(unittest.TestCase): + + def setUp(self): + Switchable.failing = False + self.client = client() + + def tearDown(self): + Switchable.failing = False + + def responses(self, headers): + """The JavaScript and the JSON responses to the same evidence.""" + return [ + self.client.get("/51Degrees.core.js", headers=headers), + self.client.post("/json", headers=headers), + ] + + def test_a_good_response_may_be_kept_by_the_browser_alone(self): + for response in self.responses({"User-Agent": CHROME}): + self.assertEqual(200, response.status_code) + self.assertEqual(CACHEABLE, response.headers.get("Cache-Control")) + etag = response.headers.get("ETag") + self.assertIsNotNone(etag, "a good response carries an ETag") + self.assertTrue(etag.startswith('"') and etag.endswith('"'), + "an ETag is a quoted string: " + etag) + self.assertIn("User-Agent", response.headers.get("Vary", "")) + + def test_the_content_type_says_which_endpoint_answered(self): + script, results = self.responses({"User-Agent": CHROME}) + self.assertEqual("application/x-javascript", + script.headers["Content-Type"]) + self.assertEqual("application/json", results.headers["Content-Type"]) + self.assertIn(b"fiftyoneDegreesManager", script.data) + self.assertEqual(42, results.get_json()["switchable"]["answer"]) + + def test_the_etag_follows_the_evidence(self): + first = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) + again = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) + other = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": SAFARI}) + self.assertEqual(first.headers["ETag"], again.headers["ETag"]) + self.assertNotEqual(first.headers["ETag"], other.headers["ETag"]) + + def test_the_etag_is_the_same_in_every_process(self): + """Python seeds the hash of a string afresh in each process, so the + two processes are given different seeds.""" + + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + code = ("import sys; sys.path.insert(0, sys.argv[1]); " + "from tests.test_web_client_side import script_etag; " + "print(script_etag())") + etags = [] + for seed in ("1", "2"): + finished = subprocess.run( + [sys.executable, "-c", code, root], capture_output=True, + text=True, cwd=root, timeout=120, + env=dict(os.environ, PYTHONHASHSEED=seed)) + self.assertEqual(0, finished.returncode, finished.stderr) + etags.append(finished.stdout.strip().splitlines()[-1]) + self.assertEqual(etags[0], etags[1]) + self.assertEqual(script_etag(), etags[0]) + + def test_revalidation_with_the_etag_is_answered_not_modified(self): + good = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) + for method in (self.client.get, self.client.head): + # A browser that revalidates sends Cache-Control as well when + # the page is reloaded, and neither header is evidence. + response = method("/51Degrees.core.js", headers={ + "User-Agent": CHROME, + "If-None-Match": good.headers["ETag"], + "Cache-Control": "max-age=0", + }) + self.assertEqual(304, response.status_code) + self.assertEqual(b"", response.data) + # The headers of the good response that RFC 9110 section 15.4.5 + # asks a 304 to repeat. + for name in ("ETag", "Cache-Control", "Vary"): + self.assertEqual(good.headers.get(name), + response.headers.get(name), name) + + def test_a_post_with_a_matching_etag_is_refused(self): + """RFC 9110 section 13.1.2 allows a 304 for GET and HEAD only, and + asks for 412 Precondition Failed for any other method.""" + + etag = self.client.post( + "/json", headers={"User-Agent": CHROME}).headers["ETag"] + response = self.client.post("/json", headers={ + "User-Agent": CHROME, "If-None-Match": etag}) + self.assertEqual(412, response.status_code) + self.assertEqual(b"", response.data) + self.assertNotIn("ETag", response.headers) + + def test_the_web_request_value_is_not_in_the_etag(self): + """The web integration may add the request itself as + server.http-request, for the usage sharing element. It holds the + request's every header, If-None-Match included, so it must not be + hashed, even though the JavaScript builder accepts every key.""" + + def with_web_request(request): + evidence = webevidence(request) + evidence["server.http-request"] = MappingProxyType(dict(evidence)) + return evidence + + webevidence = web.webevidence + with mock.patch.object(web, "webevidence", with_web_request): + etag = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) \ + .headers["ETag"] + response = self.client.get("/51Degrees.core.js", headers={ + "User-Agent": CHROME, "If-None-Match": etag}) + self.assertEqual(304, response.status_code) + + def test_a_weak_or_listed_etag_still_matches(self): + etag = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) \ + .headers["ETag"] + for value in ("W/" + etag, '"other", ' + etag, "*"): + response = self.client.get("/51Degrees.core.js", headers={ + "User-Agent": CHROME, "If-None-Match": value}) + self.assertEqual(304, response.status_code, value) + response = self.client.get("/51Degrees.core.js", headers={ + "User-Agent": CHROME, "If-None-Match": '"other"'}) + self.assertEqual(200, response.status_code) + + def test_a_response_to_a_failed_request_carries_no_etag(self): + Switchable.failing = True + for response in self.responses({"User-Agent": CHROME}): + self.assertEqual(200, response.status_code) + self.assertEqual( + NOT_CACHEABLE, response.headers.get("Cache-Control")) + self.assertNotIn("ETag", response.headers) + + def test_a_good_copy_is_kept_while_processing_fails(self): + etag = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) \ + .headers["ETag"] + Switchable.failing = True + response = self.client.get("/51Degrees.core.js", headers={ + "User-Agent": CHROME, "If-None-Match": etag}) + self.assertEqual(304, response.status_code) + + def test_a_failed_response_is_replaced_once_processing_works(self): + """The browser kept the failed response with no ETag, so it has + nothing to revalidate with and asks again in full.""" + + Switchable.failing = True + failed = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) + Switchable.failing = False + headers = {"User-Agent": CHROME} + if "ETag" in failed.headers: + headers["If-None-Match"] = failed.headers["ETag"] + response = self.client.get("/51Degrees.core.js", headers=headers) + self.assertEqual(200, response.status_code) + self.assertEqual(CACHEABLE, response.headers.get("Cache-Control")) + + +if __name__ == "__main__": + unittest.main()