diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py index 9f285f75..27a487ff 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py @@ -20,6 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter from fiftyone_pipeline_engines.engine import Engine from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue @@ -42,7 +43,7 @@ class CloudEngine(Engine): It requires the 51Degrees cloudRequestEngine to be placed in a pipeline before it. It takes that raw JSON response and parses it to extract the device part. - It also uses this data to generate a list of properties and an evidence key filter + It also uses this data to generate a list of properties. """ @@ -52,6 +53,15 @@ def __init__(self): self.datakey = "CloudEngineBase" # This should be overriden + def get_evidence_key_filter(self): + + """! + A cloud engine reads only the cloud request engine's results, so it + accepts no evidence. + """ + + return BasicListEvidenceKeyFilter([]) + def on_registration(self, pipeline): """! diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/classes/accepteverykeyelement.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/classes/accepteverykeyelement.py new file mode 100644 index 00000000..c7adc06b --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/classes/accepteverykeyelement.py @@ -0,0 +1,44 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +from fiftyone_pipeline_core.evidence_keyfilter import EvidenceKeyFilter +from fiftyone_pipeline_core.flowelement import FlowElement + + +class AcceptEveryKeyElement(FlowElement): + """! + An element that accepts every evidence key and does nothing else. A + pipeline keeps only the evidence one of its elements accepts, and the + stand-in service accepts query.User-Agent alone, so a test of how the + cloud request engine treats other evidence adds this element to keep + that evidence in the flow data. + """ + + def __init__(self): + super(AcceptEveryKeyElement, self).__init__() + self.datakey = "accepteverykey" + + def get_evidence_key_filter(self): + return EvidenceKeyFilter() + + def process_internal(self, flowdata): + pass diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_evidence.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_evidence.py new file mode 100644 index 00000000..ba9f157a --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_evidence.py @@ -0,0 +1,70 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""A cloud engine reads only the cloud request engine's results, so it +accepts no evidence, and a cloud pipeline sends the service only the +evidence the service lists and the elements the builder adds read.""" + +from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine +from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import \ + CloudRequestEngine +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase + +LICENSE_KEY = "a-licence-key" +REQUESTED = ["device.ismobile", "device.istablet"] + + +class CloudEngineEvidenceTests(CloudRequestEngineTestsBase): + + def test_a_cloud_engine_accepts_no_evidence(self): + engine = CloudEngine() + for key in ("query.user-agent", "header.user-agent", "header.x-test", + "cookie.51d_profileids", "query.session-id", + "server.client-ip"): + self.assertFalse(engine.filter_evidence_key(key), key) + + def test_a_cloud_pipeline_sends_only_the_evidence_the_service_lists(self): + """ + The stand-in service lists query.User-Agent alone, so of this + evidence only the user agent reaches it. + """ + cloud = CloudRequestEngine({ + "license_key": LICENSE_KEY, + "requested_properties": REQUESTED, + "http_client": self.mock_http(), + }) + engine = CloudEngine() + engine.datakey = "device" + + pipeline = PipelineBuilder().add(cloud).add(engine).build() + data = pipeline.create_flowdata() + data.evidence.add("query.User-Agent", "iPhone") + data.evidence.add("header.x-test", "a header") + data.evidence.add("cookie.other", "a cookie") + data.process() + + request_type, url, content = cloud.http_client.requests[-1] + self.assertEqual("iPhone", content["user-agent"]) + self.assertNotIn("x-test", content) + self.assertNotIn("other", content) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py index 649f7866..1c39b024 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py @@ -29,6 +29,7 @@ from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_pipeline_cloudrequestengine.constants import Constants +from .classes.accepteverykeyelement import AcceptEveryKeyElement from .classes.constants import Constants as TestConstants from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase @@ -111,8 +112,11 @@ def test_evidence_precedence(self, warn, evidence1, evidence2): "http_client": self.mock_http(), }) + # The stand-in service accepts query.User-Agent alone, so an element + # that accepts every key keeps the rest of the evidence for the + # engine to order. builder = PipelineBuilder() - pipeline = builder.add(engine).build() + pipeline = builder.add(AcceptEveryKeyElement()).add(engine).build() data = pipeline.create_flowdata() diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py index a5a2b8ab..786ae020 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py @@ -28,6 +28,7 @@ from fiftyone_pipeline_cloudrequestengine.constants import Constants from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from .classes.accepteverykeyelement import AcceptEveryKeyElement from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase RESOURCE_KEY = "resource_key" @@ -269,7 +270,11 @@ def test_reserved_evidence_is_left_out_of_the_body(self): "http_client": self.mock_http(), }) - pipeline = PipelineBuilder().add(engine).build() + # The stand-in service lists query.User-Agent alone, so an element + # that accepts every key keeps this evidence for the engine, which + # leaves it out whatever reaches it. + pipeline = PipelineBuilder().add(AcceptEveryKeyElement()) \ + .add(engine).build() data = pipeline.create_flowdata() data.evidence.add("query.license", "another-key") data.evidence.add("header.resource", "another-key") @@ -302,6 +307,30 @@ def test_reserved_evidence_is_left_out_of_the_body(self): for message in messages: self.assertNotIn("another-key", message) + def test_evidence_no_element_accepts_is_not_sent(self): + """ + A default pipeline keeps only the evidence one of its elements + accepts, so a header or cookie the service does not ask for is not + sent to it. + """ + engine = CloudRequestEngine({ + "license_key": LICENSE_KEY, + "requested_properties": REQUESTED, + "http_client": self.mock_http(), + }) + + pipeline = PipelineBuilder().add(engine).build() + data = pipeline.create_flowdata() + data.evidence.add("query.User-Agent", "iPhone") + data.evidence.add("header.x-test", "a header") + data.evidence.add("cookie.other", "a cookie") + data.process() + + request_type, url, content = engine.http_client.requests[-1] + self.assertEqual("iPhone", content["user-agent"]) + self.assertNotIn("x-test", content) + self.assertNotIn("other", content) + def test_properties_added_later_are_normalised_and_checked(self): """ A caller may add to the list after the engine is built, once the diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py index 73e6922d..b04553a0 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py @@ -20,7 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -from .evidence_keyfilter import EvidenceKeyFilter +from .basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter class FlowElement(object): @@ -91,14 +91,16 @@ def close(self): def get_evidence_key_filter(self): """! - Filter FlowData evidence using the FlowElement's EvidenceKeyFilter + The evidence this element accepts. An element overrides this to name + the evidence that can change its results. One that does not accepts + no evidence, so it never widens what the pipeline keeps. @rtype: EvidenceKeyFilter @return: Returns an EvidenceKeyFilter """ - return EvidenceKeyFilter() + return BasicListEvidenceKeyFilter([]) def filter_evidence(self, flowdata): diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py index bdb85c50..cb96e9c8 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py @@ -215,8 +215,10 @@ def url_encode(value): class JavaScriptBuilderEvidenceKeyFilter(EvidenceKeyFilter): - def filter(self, key): - if "query" in key: + # The query string, which the script sends back to the callback, and the + # host and protocol the callback URL is built from. + def filter_evidence_key(self, key): + if key.startswith("query."): return True if key == "header.host" or key == "header.protocol": diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index 9f2a8ca7..575508ba 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -20,6 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from .basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter from .flowelement import FlowElement from .elementdata_dictionary import ElementDataDictionary @@ -47,6 +48,15 @@ def __init__(self): self.property_cache = {} + def get_evidence_key_filter(self): + + """! + The element reads only the results of other elements, so it accepts + no evidence. + """ + + return BasicListEvidenceKeyFilter([]) + def process_internal(self, flowdata): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py index c4056232..cd85a857 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py @@ -28,7 +28,9 @@ class SequenceElementEvidenceKeyFilter(EvidenceKeyFilter): - def filter(self, key): + # The session id and the sequence. The result changes as the sequence + # grows, so they are part of the ETag. + def filter_evidence_key(self, key): if (key == "query.sequence" or key == "query.session-id"): return True diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py index 8f5402d7..083c5617 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py @@ -20,6 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from .basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter from .flowelement import FlowElement from .elementdata_dictionary import ElementDataDictionary from .messages import Messages @@ -46,6 +47,15 @@ def __init__(self): self.exclude_from_messages = True + def get_evidence_key_filter(self): + + """! + The element reads only the results of other elements, so it accepts + no evidence. + """ + + return BasicListEvidenceKeyFilter([]) + def process_internal(self, flowdata): """! diff --git a/pipeline/fiftyone_pipeline_core/tests/test_default_elements_accept.py b/pipeline/fiftyone_pipeline_core/tests/test_default_elements_accept.py new file mode 100644 index 00000000..d0ba4208 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_default_elements_accept.py @@ -0,0 +1,139 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The elements PipelineBuilder adds accept only the evidence they read, so +a default pipeline accepts what its engines read and no more. The ETag and +the Vary header of the client-side endpoints are built from what the +pipeline accepts.""" + +import unittest + +from flask import Flask, request + +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.javascriptbuilder import \ + JavascriptBuilderElement +from fiftyone_pipeline_core.jsonbundler import JSONBundlerElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_core.sequenceelement import SequenceElement +from fiftyone_pipeline_core.setheaderelement import SetHeaderElement + +from .classes.exampleflowelement1 import ExampleFlowElement1 + + +class DeclaresNothingElement(FlowElement): + """An element that does not say which evidence it accepts.""" + + def __init__(self): + super(DeclaresNothingElement, self).__init__() + self.datakey = "declaresnothing" + + def process_internal(self, flowdata): + pass + +NOT_READ = ("header.if-none-match", "header.x-test", "header.x-query", + "header.cookie", "cookie.query", "server.client-ip") + + +def default_pipeline(): + """A pipeline with the elements the builder adds and one element that + accepts header.user-agent alone.""" + return PipelineBuilder().add(ExampleFlowElement1()).build() + + +def accepted(pipeline, key): + return any(element.filter_evidence_key(key) + for element in pipeline.flow_elements) + + +class DefaultElementsAcceptTests(unittest.TestCase): + + def test_a_default_pipeline_accepts_only_what_its_elements_read(self): + pipeline = default_pipeline() + + for key in NOT_READ: + self.assertFalse(accepted(pipeline, key), key) + for key in ("header.user-agent", "query.session-id", + "query.sequence", "query.fod-js-object-name", + "header.host", "header.protocol"): + self.assertTrue(accepted(pipeline, key), key) + + def test_each_element_accepts_only_what_it_reads(self): + sequence = SequenceElement() + self.assertTrue(sequence.filter_evidence_key("query.session-id")) + self.assertTrue(sequence.filter_evidence_key("query.sequence")) + self.assertFalse(sequence.filter_evidence_key("query.other")) + + builder = JavascriptBuilderElement({}) + self.assertTrue(builder.filter_evidence_key("query.anything")) + self.assertTrue(builder.filter_evidence_key("header.host")) + self.assertTrue(builder.filter_evidence_key("header.protocol")) + self.assertFalse(builder.filter_evidence_key("header.x-query")) + self.assertFalse(builder.filter_evidence_key("cookie.query")) + + for element in (SetHeaderElement(), JSONBundlerElement()): + for key in NOT_READ + ("header.user-agent", "query.sequence"): + self.assertFalse(element.filter_evidence_key(key), + type(element).__name__ + " " + key) + + def test_processing_sets_aside_the_evidence_no_element_reads(self): + flowdata = default_pipeline().create_flowdata() + flowdata.evidence.add("header.user-agent", "a browser") + flowdata.evidence.add("header.if-none-match", '"a-tag"') + flowdata.process() + + self.assertEqual("a browser", + flowdata.evidence.get("header.user-agent")) + self.assertIsNone(flowdata.evidence.get("header.if-none-match")) + + def test_only_the_headers_the_elements_read_are_named_for_vary(self): + """The client-side endpoints name in Vary the request headers the + pipeline accepts.""" + + pipeline = default_pipeline() + with Flask(__name__).test_request_context("/", headers={ + "User-Agent": "a browser", "Accept": "*/*", + "Accept-Language": "en", "Cookie": "a=b", + "If-None-Match": '"a-tag"', "X-Test": "x"}): + named = sorted(name for name, _ in request.headers + if accepted(pipeline, "header." + name.lower())) + + self.assertEqual(["Host", "User-Agent"], named) + + + def test_an_element_that_declares_nothing_accepts_nothing(self): + element = DeclaresNothingElement() + for key in NOT_READ + ("header.user-agent", "query.session-id"): + self.assertFalse(element.filter_evidence_key(key), key) + + def test_an_element_that_declares_nothing_widens_nothing(self): + pipeline = PipelineBuilder() \ + .add(ExampleFlowElement1()) \ + .add(DeclaresNothingElement()) \ + .build() + for key in NOT_READ: + self.assertFalse(accepted(pipeline, key), key) + self.assertTrue(accepted(pipeline, "header.user-agent")) + + +if __name__ == "__main__": + unittest.main()