From 0e55650f0d138ec560705ba4b38f4e8f37a76eea Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:20:11 -0700 Subject: [PATCH] feat: Mark override-affected evaluations in analytics events Carries the override-affected marking on the evaluation event input, so the event processor keys on the marking alone and never reads the evaluation reason. An evaluation marked as override-affected produces no individual feature event and no debug event, even when the flag requests them, and it is counted in summary events like any other evaluation. The marking is part of the summary counter key, so override-affected and ordinary evaluations of the same flag, variation, and version accumulate into separate counters, and a counter that aggregates marked evaluations carries the overrideAffected marker, present only when true, like the unknown marker. The evaluator passes each prerequisite record's own marking to its event, so a marked prerequisite record produces no individual event while an unaffected prerequisite inside a marked evaluation is recorded as usual. The client passes the top-level marking to the evaluation event, keeps the marking on the reason when a migration evaluation replaces it with a wrong-type error, and presents an override-affected flag in the all-flags state with trackEvents and trackReason false and no debugEventsUntilDate, so a consumer bootstrapped from the state sends no individual events for it. The async client mirrors these changes. The OVERRIDE specification vector runner now also checks the marking each evaluation contributes to its summary counter. --- ldclient/async_client.py | 27 ++- ldclient/client.py | 27 ++- ldclient/impl/async_evaluator.py | 2 +- ldclient/impl/evaluator.py | 2 +- .../impl/events/event_processor_common.py | 11 +- ldclient/impl/events/event_summarizer.py | 10 +- ldclient/impl/events/types.py | 10 + .../impl/events/test_event_overrides.py | 188 ++++++++++++++++++ .../impl/events/test_event_summarizer.py | 6 +- .../testing/test_async_client_overrides.py | 33 +++ .../testing/test_ldclient_override_vectors.py | 12 +- ldclient/testing/test_ldclient_overrides.py | 176 +++++++++++++++- 12 files changed, 482 insertions(+), 22 deletions(-) create mode 100644 ldclient/testing/impl/events/test_event_overrides.py diff --git a/ldclient/async_client.py b/ldclient/async_client.py index d04f2ab6..4aac6f43 100644 --- a/ldclient/async_client.py +++ b/ldclient/async_client.py @@ -486,7 +486,12 @@ async def evaluate(): tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': stage, 'tracker': tracker}) - detail = EvaluationDetail(default_stage.value, None, error_reason('WRONG_TYPE')) + # The type mismatch replaces the reason. The evaluation read the same definitions, + # so the new reason keeps the override-affected marking. + reason = error_reason('WRONG_TYPE') + if detail.reason.get('overrideAffected') is True: + reason['overrideAffected'] = True + detail = EvaluationDetail(default_stage.value, None, reason) tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': default_stage, 'tracker': tracker}) @@ -546,7 +551,7 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: detail = result.detail if detail.is_default_value(): detail = EvaluationDetail(default, None, detail.reason) - self._send_event(event_factory.new_eval_event(flag, context, detail, default)) + self._send_event(event_factory.new_eval_event(flag, context, detail, default, None, result.override_affected)) return detail, flag except Exception as e: log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e))) @@ -630,13 +635,25 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState result = await self._evaluator.evaluate(flag, context, self._event_factory_default) detail = result.detail prerequisites = result.prerequisites + override_affected = result.override_affected except Exception as e: log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e))) log.debug(traceback.format_exc()) reason = {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} detail = EvaluationDetail(None, None, reason) prerequisites = [] + override_affected = False requires_experiment_data = EventFactory.is_experiment(flag, detail.reason) + track_events = flag.get('trackEvents', False) or requires_experiment_data + track_reason = requires_experiment_data + debug_events_until_date = flag.get('debugEventsUntilDate', None) + if override_affected: + # A consumer of this state sends individual events according to these fields. + # An override-affected evaluation produces no individual events, so the state + # turns them off for this flag. The flag, its value, and its reason stay. + track_events = False + track_reason = False + debug_events_until_date = None flag_state = { 'key': flag['key'], 'value': detail.value, @@ -644,9 +661,9 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState 'reason': detail.reason, 'version': flag['version'], 'prerequisites': prerequisites, - 'trackEvents': flag.get('trackEvents', False) or requires_experiment_data, - 'trackReason': requires_experiment_data, - 'debugEventsUntilDate': flag.get('debugEventsUntilDate', None), + 'trackEvents': track_events, + 'trackReason': track_reason, + 'debugEventsUntilDate': debug_events_until_date, } state.add_flag(flag_state, with_reasons, details_only_if_tracked) diff --git a/ldclient/client.py b/ldclient/client.py index f6067723..f6cf1efb 100644 --- a/ldclient/client.py +++ b/ldclient/client.py @@ -455,7 +455,12 @@ def evaluate(): tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': stage, 'tracker': tracker}) - detail = EvaluationDetail(default_stage.value, None, error_reason('WRONG_TYPE')) + # The type mismatch replaces the reason. The evaluation read the same definitions, + # so the new reason keeps the override-affected marking. + reason = error_reason('WRONG_TYPE') + if detail.reason.get('overrideAffected') is True: + reason['overrideAffected'] = True + detail = EvaluationDetail(default_stage.value, None, reason) tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': default_stage, 'tracker': tracker}) @@ -517,7 +522,7 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: detail = result.detail if detail.is_default_value(): detail = EvaluationDetail(default, None, detail.reason) - self._send_event(event_factory.new_eval_event(flag, context, detail, default)) + self._send_event(event_factory.new_eval_event(flag, context, detail, default, None, result.override_affected)) return detail, flag except Exception as e: log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e))) @@ -607,14 +612,26 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: result = self._evaluator.evaluate(flag, context, self._event_factory_default) detail = result.detail prerequisites = result.prerequisites + override_affected = result.override_affected except Exception as e: log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e))) log.debug(traceback.format_exc()) reason = {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} detail = EvaluationDetail(None, None, reason) prerequisites = [] + override_affected = False requires_experiment_data = EventFactory.is_experiment(flag, detail.reason) + track_events = flag.get('trackEvents', False) or requires_experiment_data + track_reason = requires_experiment_data + debug_events_until_date = flag.get('debugEventsUntilDate', None) + if override_affected: + # A consumer of this state sends individual events according to these fields. + # An override-affected evaluation produces no individual events, so the state + # turns them off for this flag. The flag, its value, and its reason stay. + track_events = False + track_reason = False + debug_events_until_date = None flag_state = { 'key': flag['key'], 'value': detail.value, @@ -622,9 +639,9 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: 'reason': detail.reason, 'version': flag['version'], 'prerequisites': prerequisites, - 'trackEvents': flag.get('trackEvents', False) or requires_experiment_data, - 'trackReason': requires_experiment_data, - 'debugEventsUntilDate': flag.get('debugEventsUntilDate', None), + 'trackEvents': track_events, + 'trackReason': track_reason, + 'debugEventsUntilDate': debug_events_until_date, } state.add_flag(flag_state, with_reasons, details_only_if_tracked) diff --git a/ldclient/impl/async_evaluator.py b/ldclient/impl/async_evaluator.py index 935fc84d..31f3466c 100644 --- a/ldclient/impl/async_evaluator.py +++ b/ldclient/impl/async_evaluator.py @@ -148,7 +148,7 @@ async def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: failed_prereq = prereq - event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag) + event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag, prereq_override_affected) state.add_event(event) if failed_prereq: return {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': failed_prereq.key} diff --git a/ldclient/impl/evaluator.py b/ldclient/impl/evaluator.py index f6a14a25..bbce2260 100644 --- a/ldclient/impl/evaluator.py +++ b/ldclient/impl/evaluator.py @@ -146,7 +146,7 @@ def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: EvalR # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: failed_prereq = prereq - event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag) + event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag, prereq_override_affected) state.add_event(event) if failed_prereq: return {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': failed_prereq.key} diff --git a/ldclient/impl/events/event_processor_common.py b/ldclient/impl/events/event_processor_common.py index 95ea278f..75f634b1 100644 --- a/ldclient/impl/events/event_processor_common.py +++ b/ldclient/impl/events/event_processor_common.py @@ -181,7 +181,7 @@ def make_summary_event(self, summary: EventSummary): } counters = [] for ckey, cval in flag_data.counters.items(): - variation, version = ckey + variation, version, override_affected = ckey counter = {'count': cval.count, 'value': cval.value} if variation is not None: counter['variation'] = variation @@ -189,6 +189,9 @@ def make_summary_event(self, summary: EventSummary): counter['unknown'] = True else: counter['version'] = version + # The marker is present only when true, like the unknown marker. + if override_affected: + counter['overrideAffected'] = True counters.append(counter) flag_data_out['counters'] = counters flags_out[key] = flag_data_out @@ -266,9 +269,11 @@ def _process_event(self, event: EventInput): context = event.context if not event.exclude_from_summaries: self._outbox.add_to_summary(event) - if event.track_events: + # An override-affected evaluation appears only in the summary counters. It produces + # no individual feature event and no debug event, even when the flag requests them. + if event.track_events and not event.override_affected: full_event = event - if self._should_debug_event(event): + if not event.override_affected and self._should_debug_event(event): debug_event = DebugEvent(event) elif isinstance(event, EventInputIdentify): if self._omit_anonymous_contexts: diff --git a/ldclient/impl/events/event_summarizer.py b/ldclient/impl/events/event_summarizer.py index acf3a88f..ca2b5b5e 100644 --- a/ldclient/impl/events/event_summarizer.py +++ b/ldclient/impl/events/event_summarizer.py @@ -22,10 +22,16 @@ def __repr__(self) -> str: # used only in test debugging return "EventSummaryCounter(%d, %s)" % (self.count, self.value) +# A counter is keyed by variation index, flag version, and whether the evaluations it counts +# were override-affected. The marker is part of the key, so override-affected and ordinary +# evaluations of the same flag, variation, and version accumulate into separate counters. +CounterKey = Tuple[Optional[int], Optional[int], bool] + + class EventSummaryFlag: __slots__ = ['context_kinds', 'default', 'counters'] - def __init__(self, context_kinds: Set[str], default: Any, counters: Dict[Tuple[Optional[int], Optional[int]], EventSummaryCounter]): + def __init__(self, context_kinds: Set[str], default: Any, counters: Dict[CounterKey, EventSummaryCounter]): self.context_kinds = context_kinds self.counters = counters self.default = default @@ -71,7 +77,7 @@ def summarize_event(self, event: EventInputEvaluation): if c is not None: flag_data.context_kinds.add(c.kind) - counter_key = (event.variation, None if event.flag is None else event.flag.version) + counter_key = (event.variation, None if event.flag is None else event.flag.version, event.override_affected) counter = flag_data.counters.get(counter_key) if counter is None: counter = EventSummaryCounter(1, event.value) diff --git a/ldclient/impl/events/types.py b/ldclient/impl/events/types.py index d0883c47..b107a3b7 100644 --- a/ldclient/impl/events/types.py +++ b/ldclient/impl/events/types.py @@ -52,6 +52,7 @@ class EventInputEvaluation(EventInput): "track_events", "sampling_ratio", "exclude_from_summaries", + "override_affected", ] def __init__( @@ -66,6 +67,7 @@ def __init__( default_value: Any, prereq_of: Optional[FeatureFlag] = None, track_events: bool = False, + override_affected: bool = False, ): super().__init__(timestamp, context, 1 if flag is None else flag.sampling_ratio) self.key = key @@ -79,6 +81,11 @@ def __init__( self.exclude_from_summaries = ( False if flag is None else flag.exclude_from_summaries ) + # True when a flag override affected this evaluation, directly or through a + # prerequisite or segment. The event processor keys on this scalar alone, never on the + # reason: such an evaluation produces no individual feature or debug event and is + # counted in a summary counter that carries the override-affected marker. + self.override_affected = override_affected def to_debugging_dict(self) -> dict: return { @@ -94,6 +101,7 @@ def to_debugging_dict(self) -> dict: "track_events": self.track_events, "exclude_from_summaries": self.exclude_from_summaries, "sampling_ratio": self.sampling_ratio, + "override_affected": self.override_affected, } @@ -155,6 +163,7 @@ def new_eval_event( detail: EvaluationDetail, default_value: Any, prereq_of_flag: Optional[FeatureFlag] = None, + override_affected: bool = False, ) -> EventInputEvaluation: add_experiment_data = self.is_experiment(flag, detail.reason) return EventInputEvaluation( @@ -168,6 +177,7 @@ def new_eval_event( default_value, prereq_of_flag, flag.track_events or add_experiment_data, + override_affected, ) def new_default_event( diff --git a/ldclient/testing/impl/events/test_event_overrides.py b/ldclient/testing/impl/events/test_event_overrides.py new file mode 100644 index 00000000..4f4345a3 --- /dev/null +++ b/ldclient/testing/impl/events/test_event_overrides.py @@ -0,0 +1,188 @@ +""" +Tests for how analytics events treat override-affected evaluations: they appear in summary +counters only, under a counter that carries the override-affected marker, and they produce no +individual feature event and no debug event. +""" +import json +import time +from typing import Any, Dict, List + +import pytest + +from ldclient.async_config import AsyncConfig +from ldclient.config import Config +from ldclient.context import Context +from ldclient.impl.events.async_event_processor import ( + DefaultAsyncEventProcessor +) +from ldclient.impl.events.event_processor import DefaultEventProcessor +from ldclient.impl.events.event_processor_common import EventOutputFormatter +from ldclient.impl.events.event_summarizer import ( + EventSummarizer, + EventSummaryCounter +) +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.testing.builders import FlagBuilder +from ldclient.testing.impl.events.test_async_event_processor import MockAioHttp +from ldclient.testing.stub_util import MockHttp + +context = Context.builder('userkey').name('Red').build() +timestamp = 10000 + + +def tracked_flag(key: str = 'flagkey', version: int = 2): + """A flag that requests individual feature events and debug events far into the future.""" + return FlagBuilder(key).version(version).track_events(True).debug_events_until_date(int(time.time() * 1000) + 100000).build() + + +def evaluation(flag, value: str = 'value', variation: int = 1, override_affected: bool = False, track_events: bool = True) -> EventInputEvaluation: + return EventInputEvaluation(timestamp, context, flag.key, flag, variation, value, None, 'default', None, track_events, override_affected) + + +def events_of_kind(output: List[Dict[str, Any]], kind: str) -> List[Dict[str, Any]]: + return [e for e in output if e['kind'] == kind] + + +# --------------------------------------------------------------------------- +# Summarizer and output formatter +# --------------------------------------------------------------------------- + +def test_summarizer_keeps_override_affected_counters_separate(): + flag = tracked_flag() + es = EventSummarizer() + es.summarize_event(evaluation(flag)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(evaluation(flag)) + counters = es.snapshot().flags[flag.key].counters + assert counters == { + (1, flag.version, False): EventSummaryCounter(2, 'value'), + (1, flag.version, True): EventSummaryCounter(2, 'value'), + } + + +def test_summary_output_carries_the_marker_only_on_override_affected_counters(): + flag = tracked_flag() + es = EventSummarizer() + es.summarize_event(evaluation(flag)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(EventInputEvaluation(timestamp, context, 'unknown-flag', None, None, 'default', None, 'default', None, False, False)) + output = EventOutputFormatter(Config('SDK_KEY')).make_summary_event(es.snapshot()) + + counters = output['features'][flag.key]['counters'] + assert len(counters) == 2 + by_marker = {c.get('overrideAffected'): c for c in counters} + assert by_marker[None] == {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version} + assert by_marker[True] == {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True} + assert 'overrideAffected' not in json.dumps(output['features']['unknown-flag']) + + +def test_event_input_evaluation_defaults_to_not_override_affected(): + event = EventInputEvaluation(timestamp, context, 'flag', None, None, 'default', None, 'default') + assert event.override_affected is False + assert event.to_debugging_dict()['override_affected'] is False + + +# --------------------------------------------------------------------------- +# Sync event processor +# --------------------------------------------------------------------------- + +def flush_and_get_events(ep: DefaultEventProcessor, mock_http: MockHttp) -> List[Dict[str, Any]]: + ep.flush() + ep._wait_until_inactive() + assert mock_http.request_data is not None, 'Expected to get an HTTP request but did not get one' + return json.loads(mock_http.request_data) + + +def make_processor(mock_http: MockHttp) -> DefaultEventProcessor: + return DefaultEventProcessor(Config('SDK_KEY', diagnostic_opt_out=True), mock_http) + + +def test_override_affected_evaluation_produces_no_feature_or_debug_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + output = flush_and_get_events(ep, mock_http) + + assert [e['kind'] for e in output] == ['index', 'summary'] + counters = output[1]['features'][flag.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}] + + +def test_ordinary_evaluation_still_produces_feature_and_debug_events(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag)) + output = flush_and_get_events(ep, mock_http) + + assert sorted(e['kind'] for e in output) == ['debug', 'feature', 'index', 'summary'] + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}] + + +def test_mixed_evaluations_of_one_flag_split_into_two_counters_and_one_feature_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag)) + output = flush_and_get_events(ep, mock_http) + + assert len(events_of_kind(output, 'feature')) == 1 + assert len(events_of_kind(output, 'debug')) == 1 + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert sorted(counters, key=lambda c: c['count']) == [ + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}, + {'count': 2, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}, + ] + + +def test_override_affected_prerequisite_record_produces_no_feature_event(): + parent = tracked_flag('parent', 1) + prereq = tracked_flag('prereq', 7) + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(EventInputEvaluation(timestamp, context, prereq.key, prereq, 1, 'value', None, None, parent, True, True)) + output = flush_and_get_events(ep, mock_http) + + assert [e['kind'] for e in output] == ['index', 'summary'] + counters = output[1]['features'][prereq.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': prereq.version, 'overrideAffected': True}] + + +def test_override_affected_evaluation_still_produces_an_index_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + output = flush_and_get_events(ep, mock_http) + assert output[0]['kind'] == 'index' + assert output[0]['context'] == context.to_dict() + + +# --------------------------------------------------------------------------- +# Async event processor (shares the dispatch logic) +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_async_processor_suppresses_individual_events_for_override_affected_evaluations(): + flag = tracked_flag() + mock_http = MockAioHttp() + ep = DefaultAsyncEventProcessor(AsyncConfig('SDK_KEY', diagnostic_opt_out=True), mock_http) + try: + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag)) + assert await ep.flush_and_wait(5) is True + output = json.loads(mock_http.request_data) + finally: + await ep.stop() + + assert sorted(e['kind'] for e in output) == ['debug', 'feature', 'index', 'summary'] + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert sorted(counters, key=lambda c: 'overrideAffected' in c) == [ + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}, + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}, + ] diff --git a/ldclient/testing/impl/events/test_event_summarizer.py b/ldclient/testing/impl/events/test_event_summarizer.py index d7cf4ad1..3b2a2f82 100644 --- a/ldclient/testing/impl/events/test_event_summarizer.py +++ b/ldclient/testing/impl/events/test_event_summarizer.py @@ -41,8 +41,8 @@ def test_summarize_event_increments_counters(): data = es.snapshot() expected = { - 'flag1': EventSummaryFlag({'user'}, 'default1', {(1, flag1.version): EventSummaryCounter(2, 'value1'), (2, flag1.version): EventSummaryCounter(1, 'value2')}), - 'flag2': EventSummaryFlag({'user'}, 'default2', {(1, flag2.version): EventSummaryCounter(1, 'value99')}), - 'badkey': EventSummaryFlag({'user'}, 'default3', {(None, None): EventSummaryCounter(1, 'default3')}), + 'flag1': EventSummaryFlag({'user'}, 'default1', {(1, flag1.version, False): EventSummaryCounter(2, 'value1'), (2, flag1.version, False): EventSummaryCounter(1, 'value2')}), + 'flag2': EventSummaryFlag({'user'}, 'default2', {(1, flag2.version, False): EventSummaryCounter(1, 'value99')}), + 'badkey': EventSummaryFlag({'user'}, 'default3', {(None, None, False): EventSummaryCounter(1, 'default3')}), } assert data.flags == expected diff --git a/ldclient/testing/test_async_client_overrides.py b/ldclient/testing/test_async_client_overrides.py index 7373e1db..29a17225 100644 --- a/ldclient/testing/test_async_client_overrides.py +++ b/ldclient/testing/test_async_client_overrides.py @@ -4,6 +4,7 @@ """ import asyncio import threading +import time from typing import Any, Dict, Optional import pytest @@ -12,6 +13,7 @@ from ldclient.async_config import AsyncConfig, AsyncDataSystemConfig from ldclient.context import Context from ldclient.impl.aio.concurrency import AsyncEvent +from ldclient.impl.events.types import EventInputEvaluation from ldclient.impl.integrations.files.filedata import make_flag_with_value from ldclient.testing.builders import FlagBuilder from ldclient.testing.mock_async_components import MockAsyncEventProcessor @@ -191,3 +193,34 @@ async def test_flag_value_change_listener_sees_override_value_changes(): assert change.new_value == 'override-value' finally: await client.close() + + +@pytest.mark.asyncio +async def test_override_evaluation_events_carry_override_affected_marking(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + assert await client.variation('overridden-flag', user, False) is True + records = [e for e in client._event_processor.events if isinstance(e, EventInputEvaluation)] + assert [e.key for e in records] == ['overridden-flag'] + assert records[0].override_affected is True + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_turns_off_event_tracking_for_override_affected_flags(): + overridden = FlagBuilder('overridden-flag').version(7).on(False).off_variation(0).variations(True).track_events(True).debug_events_until_date(int(time.time() * 1000) + 100000).build().to_json_dict() + plain = FlagBuilder('plain-tracked').version(1).on(False).off_variation(0).variations(True).track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + client = await make_initialized_client({'plain-tracked': plain}, source) + try: + state = await client.all_flags_state(user, with_reasons=True) + flags_state = state.to_json_dict()['$flagsState'] + assert flags_state['plain-tracked']['trackEvents'] is True + assert 'trackEvents' not in flags_state['overridden-flag'] + assert 'debugEventsUntilDate' not in flags_state['overridden-flag'] + assert flags_state['overridden-flag']['reason']['overrideAffected'] is True + assert state.get_flag_value('overridden-flag') is True + finally: + await client.close() diff --git a/ldclient/testing/test_ldclient_override_vectors.py b/ldclient/testing/test_ldclient_override_vectors.py index d9533c3d..a55ea87d 100644 --- a/ldclient/testing/test_ldclient_override_vectors.py +++ b/ldclient/testing/test_ldclient_override_vectors.py @@ -1,7 +1,8 @@ """ Runs the OVERRIDE specification test vectors. Each vector sets up LaunchDarkly data, an override layer, and an initialization state. The test evaluates one flag through the full -client stack and checks the value, the variation index, and the reason. +client stack and checks the value, the variation index, the reason, and the marking handed to the +event processor. """ import json import os @@ -11,6 +12,7 @@ from ldclient.client import Config, Context, LDClient from ldclient.datasystem import custom +from ldclient.impl.events.types import EventInputEvaluation from ldclient.impl.integrations.files.filedata import make_flag_with_value from ldclient.testing.mock_components import ( HangingSynchronizer, @@ -83,3 +85,11 @@ def test_override_spec_vector(vector: Dict[str, Any]): assert detail.value == expect['value'], "value" assert detail.variation_index == expect['variationIndex'], "variationIndex" assert_reason(expect['reason'], detail.reason) + + # summaryOverrideAffected is the marking the client hands to the event processor for this + # evaluation. The event processor keys individual-event suppression and the summary + # counter marker on that scalar, not on the reason. + if 'summaryOverrideAffected' in expect: + records = [e for e in client._event_processor._events if isinstance(e, EventInputEvaluation) and e.key == evaluate['flagKey']] + assert len(records) == 1, "expected exactly one evaluation record for the flag" + assert records[0].override_affected is expect['summaryOverrideAffected'], "summaryOverrideAffected" diff --git a/ldclient/testing/test_ldclient_overrides.py b/ldclient/testing/test_ldclient_overrides.py index 030a603f..d6d6fe3d 100644 --- a/ldclient/testing/test_ldclient_overrides.py +++ b/ldclient/testing/test_ldclient_overrides.py @@ -2,7 +2,9 @@ Tests for flag overrides through the client: the override source lifecycle, the overlay at the store read boundary, the not-initialized gate, the all-flags state, and flag change notifications. """ +import json import logging +import time from queue import Empty, Queue from typing import Any, Dict, Optional @@ -11,9 +13,13 @@ from ldclient.client import Config, Context, LDClient from ldclient.datasystem import custom from ldclient.evaluation import EvaluationDetail +from ldclient.hook import EvaluationSeriesContext, Hook, Metadata from ldclient.impl.datasystem.fdv1 import FDv1 +from ldclient.impl.events.event_processor import DefaultEventProcessor +from ldclient.impl.events.types import EventInputEvaluation from ldclient.impl.integrations.files.filedata import make_flag_with_value from ldclient.interfaces import DataSourceState, FlagChange +from ldclient.migrations import Stage from ldclient.testing.builders import ( FlagBuilder, FlagRuleBuilder, @@ -26,7 +32,7 @@ MockOverrideSource, StaticInitializer ) -from ldclient.testing.stub_util import MockEventProcessor +from ldclient.testing.stub_util import MockEventProcessor, MockHttp user = Context.create('user-key') @@ -253,3 +259,171 @@ def test_data_source_status_is_unaffected_by_overrides(): with make_uninitialized_client(source) as client: assert client.is_initialized() is False assert client.data_source_status_provider.status.state == DataSourceState.INITIALIZING + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +def evaluation_events_by_key(client: LDClient) -> Dict[str, EventInputEvaluation]: + """The evaluation records the client handed to the event processor, keyed by flag key.""" + records = {} + processor: Any = client._event_processor + for event in processor._events: + if isinstance(event, EventInputEvaluation): + records[event.key] = event + return records + + +def test_override_evaluation_events_carry_override_affected_marking(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + assert client.variation('overridden-flag', user, False) is True + records = evaluation_events_by_key(client) + assert list(records.keys()) == ['overridden-flag'] + assert records['overridden-flag'].override_affected is True + + +def test_ordinary_evaluation_events_are_not_marked(): + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'other': single_value_flag('other', True)}) + with make_initialized_client({'flag-normal': normal}, source) as client: + client.variation('flag-normal', user, 'default') + records = evaluation_events_by_key(client) + assert records['flag-normal'].override_affected is False + assert records['flag-normal'].track_events is True + + +def tracked_bool_flag(key: str) -> FlagBuilder: + return FlagBuilder(key).version(100).variations(False, True).off_variation(0).fallthrough_variation(1).track_events(True) + + +def test_overridden_prerequisite_marks_the_dependent_evaluation_records(): + # top-flag (LaunchDarkly) --> mid-flag (LaunchDarkly) --> leaf-flag (overridden) + # --> plain-flag (LaunchDarkly) + # The LaunchDarkly copy of leaf-flag is off, so the chain passes only through the override. + ld_data = { + 'top-flag': tracked_bool_flag('top-flag').on(True).prerequisite('mid-flag', 1).prerequisite('plain-flag', 1).build().to_json_dict(), + 'mid-flag': tracked_bool_flag('mid-flag').on(True).prerequisite('leaf-flag', 1).build().to_json_dict(), + 'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict(), + 'leaf-flag': tracked_bool_flag('leaf-flag').on(False).build().to_json_dict(), + } + source = MockOverrideSource(flags={'leaf-flag': tracked_bool_flag('leaf-flag').on(True).build().to_json_dict()}) + with make_initialized_client(ld_data, source) as client: + detail = client.variation_detail('top-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + records = evaluation_events_by_key(client) + assert sorted(records.keys()) == ['leaf-flag', 'mid-flag', 'plain-flag', 'top-flag'] + assert records['top-flag'].override_affected is True + assert records['mid-flag'].override_affected is True + assert records['leaf-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['mid-flag'].prereq_of is not None and records['mid-flag'].prereq_of.key == 'top-flag' + assert records['leaf-flag'].prereq_of is not None and records['leaf-flag'].prereq_of.key == 'mid-flag' + assert records['leaf-flag'].reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert records['plain-flag'].reason == {'kind': 'FALLTHROUGH'} + + +def test_all_flags_state_turns_off_event_tracking_for_override_affected_flags(): + debug_until = int(time.time() * 1000) + 100000 + ld_data = { + 'plain-tracked': FlagBuilder('plain-tracked').version(1).on(False).off_variation(0).variations(True).track_events(True).debug_events_until_date(debug_until).build().to_json_dict(), + 'dependent-tracked': FlagBuilder('dependent-tracked').version(1).on(True).variations(False, True).fallthrough_variation(1).prerequisite('overridden-flag', 0).track_events(True).debug_events_until_date(debug_until).build().to_json_dict(), + } + # The overridden flag is on and serves variation 0, so the dependent flag's prerequisite passes. + overridden = FlagBuilder('overridden-flag').version(7).on(True).fallthrough_variation(0).variations(True).track_events(True).debug_events_until_date(debug_until).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + with make_initialized_client(ld_data, source) as client: + state = client.all_flags_state(user, with_reasons=True) + assert state.valid is True + flags_state = state.to_json_dict()['$flagsState'] + + # A flag with no override keeps its tracking fields. + assert flags_state['plain-tracked']['trackEvents'] is True + assert flags_state['plain-tracked']['debugEventsUntilDate'] == debug_until + + # The overridden flag and the flag that depends on it stay in the state with their values + # and marked reasons, but with no tracking fields. + for key in ('overridden-flag', 'dependent-tracked'): + assert flags_state[key]['reason']['overrideAffected'] is True, key + assert 'trackEvents' not in flags_state[key], key + assert 'trackReason' not in flags_state[key], key + assert 'debugEventsUntilDate' not in flags_state[key], key + assert state.get_flag_value('overridden-flag') is True + assert state.get_flag_value('dependent-tracked') is True + assert flags_state['overridden-flag']['version'] == 7 + + +def test_all_flags_state_keeps_details_of_override_affected_flags_when_details_only_for_tracked_flags(): + # With details only for tracked flags, an override-affected flag counts as untracked, so its + # version and reason are omitted like any other untracked flag, and its value stays. + overridden = FlagBuilder('overridden-flag').version(7).on(False).off_variation(0).variations(True).track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + with make_initialized_client({}, source) as client: + state = client.all_flags_state(user, with_reasons=True, details_only_for_tracked_flags=True) + flags_state = state.to_json_dict()['$flagsState'] + assert state.get_flag_value('overridden-flag') is True + assert 'version' not in flags_state['overridden-flag'] + assert 'reason' not in flags_state['overridden-flag'] + + +def test_wrong_type_result_of_overridden_flag_stays_marked(): + details = [] + + class CapturingHook(Hook): + @property + def metadata(self) -> Metadata: + return Metadata(name='capturing-hook') + + def before_evaluation(self, series_context: EvaluationSeriesContext, data: dict) -> dict: + return data + + def after_evaluation(self, series_context: EvaluationSeriesContext, data: dict, detail: EvaluationDetail) -> dict: + details.append(detail) + return data + + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', 'not-a-stage')}) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor, hooks=[CapturingHook()]) + with LDClient(config, start_wait=0) as client: + stage, _ = client.migration_variation('overridden-flag', user, Stage.OFF) + assert stage == Stage.OFF + assert len(details) == 1 + assert details[0].value == 'off' + assert details[0].reason == {'kind': 'ERROR', 'errorKind': 'WRONG_TYPE', 'overrideAffected': True} + + +def test_override_affected_evaluations_appear_only_in_summary_output(): + # End to end through the real event processor: the overridden flag requests individual + # feature events and debug events, and an ordinary flag requests feature events. + debug_until = int(time.time() * 1000) + 100000 + overridden = FlagBuilder('flag-tracked-override').version(300).on(False).off_variation(0).variations('override-value').track_events(True).debug_events_until_date(debug_until).build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'flag-tracked-override': overridden}) + mock_http = MockHttp() + + initializer = StaticInitializer({'flag-normal': normal}, {}) + datasystem = custom().initializers([initializer.builder]).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, diagnostic_opt_out=True, event_processor_class=lambda config: DefaultEventProcessor(config, mock_http)) + with LDClient(config, start_wait=5) as client: + assert client.is_initialized() is True + for _ in range(2): + assert client.variation('flag-tracked-override', user, 'default1') == 'override-value' + assert client.variation('flag-normal', user, 'default2') == 'normal-value' + client.flush() + client._event_processor._wait_until_inactive() + + assert mock_http.request_data is not None + output = json.loads(mock_http.request_data) + kinds = sorted(e['kind'] for e in output) + assert kinds == ['feature', 'index', 'summary'] + feature = [e for e in output if e['kind'] == 'feature'][0] + assert feature['key'] == 'flag-normal' + summary = [e for e in output if e['kind'] == 'summary'][0] + assert summary['features']['flag-tracked-override']['default'] == 'default1' + assert summary['features']['flag-tracked-override']['counters'] == [ + {'count': 2, 'value': 'override-value', 'variation': 0, 'version': 300, 'overrideAffected': True} + ] + assert summary['features']['flag-normal']['counters'] == [{'count': 1, 'value': 'normal-value', 'variation': 0, 'version': 100}]