diff --git a/src/__tests__/context.test.js b/src/__tests__/context.test.js index 4dacff5..b5a400a 100644 --- a/src/__tests__/context.test.js +++ b/src/__tests__/context.test.js @@ -2889,6 +2889,3008 @@ describe("Context", () => { }); }); + describe("holdouts", () => { + // All fixtures below are ported verbatim (field-for-field) from the real cross-sdk-tests + // scenario battery at ~/git_tree/sdks/cross-sdk-tests/test_scenarios_complete.json, indices + // 202-221 (scenarios 203-222). Each it() names the real scenario number it ports. Two + // scenarios in that range are deliberately not ported 1:1 as separate it() blocks: + // - 209 ("Old-Payload Tolerance For Uncovered Experiment") is an inertness baseline that + // must pass identically on a holdout-unaware SDK; it exercises no holdout-specific code + // path beyond what scenario 206 (opt-in coverage) already covers here. + // - 212 ("Independent Holdouts Across Experiments") exercises the same + // held-out/not-held-out-with-independent-seeds shape already covered by the combination + // of scenarios 203 and 204 below, just spread across two experiments instead of one. + const buildHoldoutResponse = (experiments, holdouts) => ({ experiments, holdouts }); + + // Scenario 203 (index 202): a unit landing in a holdout's variant 0 gets control for the + // experiment it covers (no exposure of its own) and exactly one exposure for the holdout. + it("suppresses the covered experiment and fires only the holdout's exposure when held out (scenario 203)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_held_out", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_a", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_holdout_held_out")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_a", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 204 (index 203): a unit outside the holdout's suppressed arm assigns to the + // covered experiment exactly as it would without holdouts, plus one exposure for the + // holdout carrying its nonzero (not-held-out) variant — holdouts always fire on first + // evaluation, not only when they actually suppress. + it("assigns normally and still fires the holdout's own exposure when not held out (scenario 204)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_not_held_out", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_a", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const notHeldOutParams = { units: { session_id: "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3" } }; + const context = new Context(sdk, contextOptions, notHeldOutParams, response); + expect(context.treatment("exp_holdout_not_held_out")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "exp_holdout_not_held_out", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 11, + name: "holdout_a", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 205 (index 204): an experiment covered by two holdouts is suppressed if the unit + // is held out by EITHER one (union semantics). Pinned so the low-id holdout does NOT hold the + // unit out and the high-id holdout does, ruling out an implementation that only ever consults + // holdouts[0]. Both holdouts still emit their own independent exposure. + it("suppresses via union when the higher-id holdout holds the unit out (scenario 205)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_union", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11, 12], + }, + ], + [ + { + id: 11, + name: "holdout_low_id", + iteration: 1, + unitType: "session_id", + seedHi: 1, + seedLo: 222, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + { + id: 12, + name: "holdout_high_id", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_holdout_union")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_low_id", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 12, + name: "holdout_high_id", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 206 (index 205): holdout coverage never leaks to a sibling experiment lacking + // holdoutIds, even when that sibling shares the same unit and context as an experiment the + // same holdout suppresses. + it("does not leak suppression to a sibling experiment with no holdoutIds (scenario 206)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_covered", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + { + id: 2, + name: "exp_holdout_uncovered_sibling", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + }, + ], + [ + { + id: 11, + name: "holdout_a", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_holdout_covered")).toEqual(0); + expect(context.treatment("exp_holdout_uncovered_sibling")).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_a", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 2, + name: "exp_holdout_uncovered_sibling", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 207 (index 206): a holdoutIds entry with no matching holdouts[] element is + // silently ignored - wire inconsistency tolerance, not an error - while a second, valid id in + // the same list still applies normally. + it("tolerates a dangling holdoutId while a valid one in the same list still applies (scenario 207)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_dangling_plus_valid", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [999, 11], + }, + ], + [ + { + id: 11, + name: "holdout_a", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + // Fixed per final-review finding M-4: the construction and the async publish + // assertions are split apart. `.not.toThrow()` only catches SYNCHRONOUS throws, so + // wrapping the `.then()` callback (with its `done()` call) inside it let any assertion + // failure inside the callback escape as an unhandled rejection / test timeout instead + // of a clean, attributable test failure. The synchronous construction is asserted not + // to throw on its own, and the publish/exposure assertions run unwrapped afterward, + // matching how every other test in this block is structured. + expect(() => new Context(sdk, contextOptions, contextParams, response)).not.toThrow(); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_holdout_dangling_plus_valid")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_a", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 208 (index 207): a holdout suppresses a full-on experiment (fullOnVariant != 0) + // exactly as it does a traffic-eligible one - explicit holdoutIds coverage, not full-on + // status, decides suppression. + it("suppresses a full-on experiment regardless of its fullOnVariant (scenario 208)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_a", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_holdout_fullon")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_a", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 210 (index 209): an explicit override wins over holdout membership for the + // experiment's own assignment; the override path still evaluates coverage, so the holdout's + // own exposure fires alongside it. + it("lets override win the variant while the holdout still fires its own exposure (scenario 210)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_override", + iteration: 1, + unitType: "session_id", + seedHi: 3603515, + seedLo: 233373850, + split: [0.5, 0.5], + trafficSeedHi: 449867249, + trafficSeedLo: 455443629, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [1], + }, + ], + [ + { + id: 1, + name: "holdout_override_precedence", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + context.override("exp_holdout_override", 1); + expect(context.treatment("exp_holdout_override")).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "exp_holdout_override", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: false, + eligible: true, + overridden: true, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 1, + name: "holdout_override_precedence", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 211 (index 210): holdout membership forces control and suppresses the + // experiment's own exposure even with a custom assignment on file; only the holdout's own + // exposure fires. + it("lets holdout suppression beat a custom assignment (scenario 211)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_custom", + iteration: 1, + unitType: "session_id", + seedHi: 3603515, + seedLo: 233373850, + split: [0.5, 0.5], + trafficSeedHi: 449867249, + trafficSeedLo: 455443629, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [1], + }, + ], + [ + { + id: 1, + name: "holdout_custom_precedence", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + context.customAssignment("exp_holdout_custom", 1); + expect(context.treatment("exp_holdout_custom")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "holdout_custom_precedence", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 213 (index 212): the same unit and shared holdout definition produce consistent + // holdout membership across two different covered experiments; the shared holdout's own + // exposure fires once on first evaluation and neither covered experiment fires its own + // exposure — the second call must fire nothing at all, not even a duplicate holdout exposure. + it("fires a shared holdout's exposure once across two covered experiments (scenario 213)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_shared_a", + iteration: 1, + unitType: "session_id", + seedHi: 3603515, + seedLo: 233373850, + split: [0.5, 0.5], + trafficSeedHi: 449867249, + trafficSeedLo: 455443629, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [1], + }, + { + id: 2, + name: "exp_shared_b", + iteration: 1, + unitType: "session_id", + seedHi: 3603515, + seedLo: 233373850, + split: [0.5, 0.5], + trafficSeedHi: 449867249, + trafficSeedLo: 455443629, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [1], + }, + ], + [ + { + id: 1, + name: "holdout_shared", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.treatment("exp_shared_a")).toEqual(0); + expect(publisher.publish.mock.calls.length).toEqual(0); // nothing published yet; check pending directly + expect(context.pending()).toEqual(1); + + expect(context.treatment("exp_shared_b")).toEqual(0); + // exp_shared_b is also suppressed, and the shared holdout already fired for exp_shared_a — + // no further exposures of any kind should be queued. + expect(context.pending()).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + // Tightened per final-review finding M-3: assert the exact exposures array (matching + // every other test in this describe block) so this verifies the SPECIFIC exposure that + // fired is the shared holdout's own — not merely that exactly one exposure fired, which + // would pass even if the wrong exposure (e.g. exp_shared_a's own) had fired instead. + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "holdout_shared", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenarios 214-220 (indices 213-219): the three-arm holdout battery. Same shared unit + // (e791e240fcd3df7d238cfc285f475e8152fcc0ec) throughout; each scenario pins a distinct arm + // semantic. + describe("holdout arms battery", () => { + // Scenario 214 (index 213): a 3-arm holdout's variant 0 forces control for every covered + // experiment, full-on or not, identically to a 2-arm holdout's variant 0; the holdout's + // own exposure fires once at variant 0 and neither covered experiment emits its own + // exposure. + it("3-arm variant 0 holds out both full-on and non-full-on covered experiments (scenario 214)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_three_arm_0_non_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [21], + }, + { + id: 2, + name: "exp_three_arm_0_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [21], + }, + ], + [ + { + id: 21, + name: "holdout_three_arm_v0", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 1, + split: [0.3, 0.3, 0.4], + holdoutType: "all_full_on", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.treatment("exp_three_arm_0_non_fullon")).toEqual(0); + expect(context.pending()).toEqual(1); + + expect(context.treatment("exp_three_arm_0_fullon")).toEqual(0); + expect(context.pending()).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 21, + name: "holdout_three_arm_v0", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 215 (index 214): a 3-arm holdout's variant 1 forces control only for a covered + // experiment with fullOnVariant==0; a covered full-on experiment (fullOnVariant!=0) takes + // its normal assignment path and is assigned its own fullOnVariant with fullOn=true, not + // suppressed. + it("3-arm variant 1 holds out only the non-full-on experiment (scenario 215)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_three_arm_1_non_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [22], + }, + { + id: 2, + name: "exp_three_arm_1_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [22], + }, + ], + [ + { + id: 22, + name: "holdout_three_arm_v1", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 3, + split: [0.3, 0.3, 0.4], + holdoutType: "all_full_on", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_three_arm_1_non_fullon")).toEqual(0); + expect(context.treatment("exp_three_arm_1_fullon")).toEqual(2); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 22, + name: "holdout_three_arm_v1", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 2, + name: "exp_three_arm_1_fullon", + unit: "session_id", + exposedAt: timeOrigin, + variant: 2, + assigned: true, + eligible: true, + overridden: false, + fullOn: true, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 216 (index 215): a 3-arm holdout's variant 2 (normal traffic) evaluates every + // covered experiment exactly as if uncovered, full-on or not; the holdout's own exposure + // fires once at variant 2. + it("3-arm variant 2 defers to the normal path for every covered experiment (scenario 216)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_three_arm_2_non_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [23], + }, + { + id: 2, + name: "exp_three_arm_2_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [23], + }, + ], + [ + { + id: 23, + name: "holdout_three_arm_v2", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 0, + split: [0.3, 0.3, 0.4], + holdoutType: "all_full_on", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_three_arm_2_non_fullon")).toEqual(1); + expect(context.treatment("exp_three_arm_2_fullon")).toEqual(2); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "exp_three_arm_2_non_fullon", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 23, + name: "holdout_three_arm_v2", + unit: "session_id", + exposedAt: timeOrigin, + variant: 2, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 2, + name: "exp_three_arm_2_fullon", + unit: "session_id", + exposedAt: timeOrigin, + variant: 2, + assigned: true, + eligible: true, + overridden: false, + fullOn: true, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 217 (index 216): a full-on experiment covered by a 3-arm holdout's variant 1 is + // not short-circuited to its fullOnVariant: audienceStrict is evaluated first, so an + // audience mismatch still forces control (not suppression) with assigned=false and + // audienceMismatch=true; the experiment's own exposure fires because it was evaluated and + // rejected by audience, not held out. + it("3-arm variant 1 defers to the normal path for an audience mismatch (scenario 217)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_three_arm_1_audience_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: JSON.stringify({ filter: [{ gte: [{ var: "age" }, { value: 20 }] }] }), + audienceStrict: true, + customFieldValues: null, + holdoutIds: [24], + }, + ], + [ + { + id: 24, + name: "holdout_three_arm_v1", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 3, + split: [0.3, 0.3, 0.4], + holdoutType: "all_full_on", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + context.attribute("age", 5); + expect(context.treatment("exp_three_arm_1_audience_fullon")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "exp_three_arm_1_audience_fullon", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: false, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: true, + ruleOverride: false, + }, + { + id: 24, + name: "holdout_three_arm_v1", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 218 (index 217): the same full-on/audience-mismatch experiment, covered + // instead by the 3-arm holdout's variant 2 (normal traffic), yields the identical verdict + // as scenario 217's variant 1: control, assigned=false, audienceMismatch=true. Pins that + // arm 1 defers to the normal assignment path instead of short-circuiting to the full-on + // variant. + it("3-arm variant 2 reaches the identical audience-mismatch verdict as variant 1 (scenario 218)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_three_arm_2_audience_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: JSON.stringify({ filter: [{ gte: [{ var: "age" }, { value: 20 }] }] }), + audienceStrict: true, + customFieldValues: null, + holdoutIds: [25], + }, + ], + [ + { + id: 25, + name: "holdout_three_arm_v2", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 0, + split: [0.3, 0.3, 0.4], + holdoutType: "all_full_on", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + context.attribute("age", 5); + expect(context.treatment("exp_three_arm_2_audience_fullon")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "exp_three_arm_2_audience_fullon", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: false, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: true, + ruleOverride: false, + }, + { + id: 25, + name: "holdout_three_arm_v2", + unit: "session_id", + exposedAt: timeOrigin, + variant: 2, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 219 (index 218): a covered experiment is suppressed if EITHER applicable + // holdout holds it out, even when a 3-arm holdout's own arm (variant 2, normal traffic) + // would not have held it out on its own; both holdouts still emit their own independent + // exposure with their own variant. A second experiment covered only by the + // non-suppressing 3-arm holdout takes its normal assignment path and emits its own + // exposure, proving that holdout 22's arm is not itself suppressing. + it("union: a 2-arm holdout holds out while the applicable 3-arm holdout does not (scenario 219)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_union_two_arm_wins", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11, 22], + }, + { + id: 2, + name: "exp_only_three_arm_normal_path", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [22], + }, + ], + [ + { + id: 11, + name: "holdout_union_two_arm", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + }, + { + id: 22, + name: "holdout_union_three_arm", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 0, + split: [0.3, 0.3, 0.4], + holdoutType: "all_full_on", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_union_two_arm_wins")).toEqual(0); + expect(context.treatment("exp_only_three_arm_normal_path")).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_union_two_arm", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 22, + name: "holdout_union_three_arm", + unit: "session_id", + exposedAt: timeOrigin, + variant: 2, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 2, + name: "exp_only_three_arm_normal_path", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 220 (index 219): union, the other direction: a 3-arm holdout's variant 1 holds + // a non-full-on covered experiment out even when an applicable 2-arm holdout alone would + // not have; both holdouts still emit their own independent exposure with their own + // variant. A second experiment covered only by the non-suppressing 2-arm holdout takes its + // normal assignment path and emits its own exposure, proving that holdout 11's arm is not + // itself suppressing. + it("union: a 3-arm holdout holds out while the applicable 2-arm holdout does not (scenario 220)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_union_three_arm_wins", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11, 22], + }, + { + id: 2, + name: "exp_only_two_arm_normal_path", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_union_two_arm", + iteration: 1, + unitType: "session_id", + seedHi: 1, + seedLo: 222, + split: [0.1, 0.9], + }, + { + id: 22, + name: "holdout_union_three_arm", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 3, + split: [0.3, 0.3, 0.4], + holdoutType: "all_full_on", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_union_three_arm_wins")).toEqual(0); + expect(context.treatment("exp_only_two_arm_normal_path")).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_union_two_arm", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 22, + name: "holdout_union_three_arm", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 2, + name: "exp_only_two_arm_normal_path", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + }); + + // Scenario 221 (index 220): a covered full-on experiment can resolve before its unit type is + // installed; after unit() supplies that unit, resolving it again publishes the holdout + // exposure that was unavailable during the first evaluation. This is a regression test for + // the Task 6 fix round (C-1): a real bug existed here (the holdout exposure was permanently + // lost) and was fixed via _invalidateAssignmentsPinnedWithMissingUnit plus a change to + // _unitHash's negative-result caching. + it("publishes the previously-unavailable holdout exposure once its late unit is set (scenario 221)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_late_unit_fullon", + iteration: 1, + unitType: "user_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5, 0.0], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_late_user_id", + iteration: 1, + unitType: "user_id", + seedHi: 1, + seedLo: 222, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const lateUnitParams = { units: { session_id: "context-created-without-user-id" } }; + const context = new Context(sdk, contextOptions, lateUnitParams, response); + + // Before the covered unit type is installed: resolves via the full-on path (no unit + // needed for that), the holdout can't be resolved yet, and peek() fires no exposures. + expect(context.peek("exp_holdout_late_unit_fullon")).toEqual(2); + expect(context.pending()).toEqual(0); + + context.unit("user_id", "e791e240fcd3df7d238cfc285f475e8152fcc0ec"); + + expect(context.treatment("exp_holdout_late_unit_fullon")).toEqual(2); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "exp_holdout_late_unit_fullon", + unit: "user_id", + exposedAt: timeOrigin, + variant: 2, + assigned: true, + eligible: true, + overridden: false, + fullOn: true, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 11, + name: "holdout_late_user_id", + unit: "user_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 222 (index 221): identical to scenario 214 (three-arm variant 0), but the holdout + // definition omits the holdoutType field entirely. An SDK must derive the holdout's arm count + // from its split length alone, so suppression of both the full-on and non-full-on covered + // experiments is unchanged. Pins that holdoutType is not required on the wire and arm count + // is never read from it. + it("derives 3-arm arity from split length alone, with no holdoutType field on the wire (scenario 222)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_three_arm_0_non_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [21], + }, + { + id: 2, + name: "exp_three_arm_0_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [21], + }, + ], + [ + { + id: 21, + name: "holdout_arity_from_split", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 1, + split: [0.3, 0.3, 0.4], + // deliberately no holdoutType field + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_three_arm_0_non_fullon")).toEqual(0); + expect(context.treatment("exp_three_arm_0_fullon")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 21, + name: "holdout_arity_from_split", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // A matching assignmentRules rule (a JS-SDK-only feature) wins over holdout suppression, + // the same way an explicit override() does: it's an author-specified assignment (flagged + // `ruleOverride`, excluded from stats), not the experiment's own randomized/custom path + // that suppression is meant to gate. The holdout fixture (id 11, seedHi 13, seedLo 111, + // split [0.1, 0.9]) is identical to scenario 203's holdout_a, which is already proven to + // hold out the default `contextParams` unit — proving suppression is genuinely in play + // here and the rule is what overrides it, not merely the holdout never applying. + it("lets a matching assignment rule's variant win over holdout suppression", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_rules", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + assignmentRules: JSON.stringify({ + rules: [ + { + name: "US Internal Users", + type: "assign", + conditions: { and: [{ eq: [{ var: "country" }, { value: "US" }] }] }, + environments: [], + variant: 1, + }, + ], + }), + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_rules_suppression", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + context.attribute("country", "US"); + + expect(context.treatment("exp_holdout_rules")).toEqual(1); + + // `ruleOverride` is set and `assigned` stays false (a rule match is not a randomized + // assignment), matching the semantics of every other rule-matched assignment. + const assignment = context._assignments["exp_holdout_rules"]; + expect(assignment.ruleOverride).toEqual(true); + expect(assignment.assigned).toEqual(false); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "exp_holdout_rules", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: false, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: true, + }, + { + id: 11, + name: "holdout_rules_suppression", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Final-review Finding I-2, Test A: regression coverage for Task 5's override-fast-path + // fix (commit 716430c). Before that fix, the `hasOverride` branch in `_assign()` returned + // the cached assignment early on `overridden && variant match` alone, without checking + // `holdoutSetMatches` — so an already-overridden experiment whose holdout coverage changed + // across a refresh (here: holdout coverage added where there was none) kept returning the + // same frozen assignment forever, and the newly-applicable holdout's own exposure could + // never fire. This constructs exactly that: cache an overridden assignment with NO holdout + // coverage, refresh so the SAME experiment now has holdoutIds, and confirm the holdout's + // own exposure fires on the next treatment() call (proving the fast path rebuilt rather + // than returning the stale pinned holdouts/holdoutAssignments from before the refresh). + it("Task 5 regression: override fast path revalidates the holdout set added by a refresh (I-2 Test A)", (done) => { + const baseExperiment = { + id: 1, + name: "exp_override_holdout_added", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + }; + + const initialResponse = buildHoldoutResponse([baseExperiment], []); + + const holdout = { + id: 11, + name: "holdout_added_after_override", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }; + + const refreshedResponse = buildHoldoutResponse([{ ...baseExperiment, holdoutIds: [11] }], [holdout]); + + const context = new Context(sdk, contextOptions, contextParams, initialResponse); + context.override("exp_override_holdout_added", 1); + + // Caches the overridden assignment with no holdout coverage yet. + expect(context.treatment("exp_override_holdout_added")).toEqual(1); + expect(context.pending()).toEqual(1); + + provider.getContextData.mockReturnValue(Promise.resolve(refreshedResponse)); + + context.refresh().then(() => { + // Pre-fix: this returns the stale cached assignment (holdouts: undefined) without + // ever resolving the newly-applicable holdout, so its own exposure never fires. + // Override still wins the variant either way (scenario 210's precedent) — the + // regression is specifically about the holdout's own exposure never being able to + // fire once coverage is added after the override was cached. + expect(context.treatment("exp_override_holdout_added")).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + const exposures = publisher.publish.mock.calls[0][0].exposures; + const holdoutExposure = exposures.find((e) => e.name === "holdout_added_after_override"); + + expect(holdoutExposure).toMatchObject({ + id: 11, + unit: "session_id", + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }); + done(); + }); + }); + }); + + // Final-review Finding I-2, Test B: regression coverage for Task 5's holdout-arm-count + // pinning fix (commit 716430c). Before that fix, suppression read the arm count live from + // `holdouts[i].data.split.length` instead of the pinned `holdoutArmCount` on the holdout's + // own cached Assignment — but `_getHoldoutAssignment`'s cache only invalidates on (id, + // iteration) change, so a same-iteration split-length change (not expected on the real + // wire, but exactly the edge case the fix protects against) could desync the live arm + // count from the arm count the resolved variant was actually interpreted against. + // + // Construct: resolve a 2-arm holdout for one experiment (pinning holdoutArmCount: 2 and + // variant: 1, both cached independently of any one covered experiment). Refresh so the + // SAME holdout (same id, same iteration) now reports a 3-arm split. Query a SECOND, + // previously-unqueried experiment (fullOnVariant: 0, the arm-1-specific case per + // isHeldOutBy) covered by the same holdout: its own assignment is built fresh, so it must + // resolve the holdout via `_getHoldoutAssignment`, hitting the (id, iteration) cache seeded + // by the first experiment. The seeds are chosen so the two arm counts disagree on the + // verdict: isHeldOutBy(1, 2, fullOnVariant=0) is false (2-arm, non-suppressing) but + // isHeldOutBy(1, 3, fullOnVariant=0) is true (3-arm variant 1 forces control when + // fullOnVariant===0) — so if the live (3) arm count were used instead of the pinned (2) + // one, the second experiment would flip from "assigned normally" to "suppressed + // (assigned: false, no own exposure)". Asserting it stays assigned normally (with its own + // exposure firing) after the refresh proves the pinned value governs, not the live one. + it("Task 5 regression: same-iteration holdout split-length change does not desync the pinned arm count (I-2 Test B)", (done) => { + const holdoutTwoArm = { + id: 11, + name: "holdout_pin_test", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }; + + const expA = { + id: 1, + name: "exp_pin_seed_a", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }; + + const initialResponse = buildHoldoutResponse([expA], [holdoutTwoArm]); + + // Same unit hashed by scenario 204 to resolve this identical holdout definition + // (seedHi 13, seedLo 111, split [0.1, 0.9]) to variant 1 — i.e. NOT held out under a + // 2-arm interpretation. + const pinParams = { units: { session_id: "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3" } }; + const context = new Context(sdk, contextOptions, pinParams, initialResponse); + + // Seeds _getHoldoutAssignment's cache: resolves and pins holdoutArmCount: 2 (the split + // length live at this moment) alongside variant: 1. Matches scenario 204's verdict for + // the identical holdout+unit pairing (not held out under 2 arms). + expect(context.treatment("exp_pin_seed_a")).toEqual(0); + + // fullOnVariant: 0 is the key choice here — isHeldOutBy's 3-arm/variant-1 rule only + // forces control when the covered experiment's fullOnVariant === 0, so this is the + // only shape where the pinned (2) vs. live (3) arm count actually disagree on the + // suppression verdict. + const expB = { + ...expA, + id: 2, + name: "exp_pin_seed_b", + }; + + // Same holdout id AND iteration — the key precondition — but now 3 arms instead of 2. + const holdoutThreeArm = { ...holdoutTwoArm, split: [0.3, 0.3, 0.4] }; + + const refreshedResponse = buildHoldoutResponse([expA, expB], [holdoutThreeArm]); + + provider.getContextData.mockReturnValue(Promise.resolve(refreshedResponse)); + + context.refresh().then(() => { + // exp_pin_seed_b is queried for the first time here, so its own assignment is built + // fresh and must resolve the holdout via _getHoldoutAssignment, hitting the (id, + // iteration) cache seeded above. + // Pinned arm count (2, correct): isHeldOutBy(1, 2, 0) is false -> NOT suppressed + // -> normal traffic-split assignment path -> assigned: true, own exposure fires. + // Live arm count (3, the pre-fix bug): isHeldOutBy(1, 3, 0) is true -> suppressed + // -> assigned: false, variant forced to 0, own exposure does NOT fire. + context.treatment("exp_pin_seed_b"); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + const exposures = publisher.publish.mock.calls[0][0].exposures; + const ownExposure = exposures.find((e) => e.name === "exp_pin_seed_b"); + + expect(ownExposure).toMatchObject({ + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }); + done(); + }); + }); + }); + + // This behavior is inherited from java-sdk's Context (Context.java:325-381, + // invalidateAssignmentsPinnedWithMissingUnit) and is intentionally left unchanged here for + // parity. Documented as known behavior, not a target for a JS-only fix. + describe("holdouts: known java-sdk-parity behavior (not fixed here)", () => { + // A full-on experiment's own variant never needs a unit, so treatment() can resolve and + // expose it before the covered experiment's unit type is ever installed. If a holdout + // covering it also needs that unit, its own assignment resolves to null and gets pinned + // into the (now-exposed) assignment's holdoutAssignments. Once installed, `unit()`'s + // eviction only clears UNEXPOSED assignments (to avoid double-firing the covered + // experiment's own exposure — see the next test) — so this assignment, already exposed, + // is never evicted, and the holdout's null entry can never be repaired. The holdout's own + // exposure is lost for the life of the context. java-sdk has the identical gap. + it("permanently loses a holdout's exposure when treatment() (not peek()) resolves a full-on experiment before its unit is set", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_treatment_before_unit", + iteration: 1, + unitType: "user_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5, 0.0], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_treatment_before_unit", + iteration: 1, + unitType: "user_id", + seedHi: 1, + seedLo: 222, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const lateUnitParams = { units: { session_id: "context-created-without-user-id" } }; + const context = new Context(sdk, contextOptions, lateUnitParams, response); + + // treatment() (not peek()) fires the covered experiment's own exposure immediately, + // since full-on doesn't need a unit — pinning a null holdout entry as a side effect. + expect(context.treatment("exp_holdout_treatment_before_unit")).toEqual(2); + expect(context.pending()).toEqual(1); + + context.unit("user_id", "e791e240fcd3df7d238cfc285f475e8152fcc0ec"); + + expect(context.treatment("exp_holdout_treatment_before_unit")).toEqual(2); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + const exposures = publisher.publish.mock.calls[0][0].exposures; + // Known limitation: only the covered experiment's own exposure is ever published. + // The holdout's exposure never fires, even though its unit is now available. + expect(exposures).toEqual([ + { + id: 1, + name: "exp_holdout_treatment_before_unit", + unit: "user_id", + exposedAt: timeOrigin, + variant: 2, + assigned: true, + eligible: true, + overridden: false, + fullOn: true, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // This behavior is inherited from java-sdk's Context (Context.java:974-1005, + // experimentMatches/holdoutSetMatches) and is intentionally left unchanged here for + // parity. Documented as known behavior, not a target for a JS-only fix. + // + // experimentMatches() folds holdoutSetMatches() into its cache-validity check, so ANY + // change to an experiment's applicable-holdout set — even adding a holdout that does not + // end up suppressing the unit — invalidates the cached assignment and forces a full + // rebuild via _assign(), which starts the new Assignment with exposed: false regardless + // of whether the resolved variant/suppression outcome actually changed. If the covered + // experiment had already fired its own exposure before the refresh, the next treatment() + // call fires it again. java-sdk has the identical gap: the covered-experiment side of + // this exact scenario is untested there for the non-override path, but the override path + // has an equivalent, explicitly-asserted duplicate (see + // holdoutBecomingApplicableAfterRefreshStillFiresForOverriddenExperiment in + // ContextHoldoutTest.java), governed by the same experimentMatches/holdoutSetMatches + // mechanism. + it("duplicates the covered experiment's own exposure when a refresh adds a non-suppressing holdout to an already-exposed experiment", (done) => { + const baseExperiment = { + id: 1, + name: "exp_holdout_added_non_suppressing", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 1, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + }; + + const initialResponse = buildHoldoutResponse([baseExperiment], []); + + const context = new Context(sdk, contextOptions, contextParams, initialResponse); + + // Fires the covered experiment's own exposure — no holdout coverage yet. + expect(context.treatment("exp_holdout_added_non_suppressing")).toEqual(1); + expect(context.pending()).toEqual(1); + + const nonSuppressingHoldout = { + id: 11, + name: "holdout_non_suppressing", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + // This unit always lands in variant 1 of this holdout, which never holds out a + // full-on-variant-1 experiment (isHeldOutBy only suppresses on variant 0, or + // variant 1 of a 3-arm holdout when fullOnVariant === 0). + split: [0.0, 1.0], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }; + + const refreshedResponse = buildHoldoutResponse( + [{ ...baseExperiment, holdoutIds: [11] }], + [nonSuppressingHoldout] + ); + + provider.getContextData.mockReturnValue(Promise.resolve(refreshedResponse)); + + context.refresh().then(() => { + // Known limitation: the covered experiment's outcome hasn't changed (still + // variant 1, still not suppressed), but the holdout-set change alone forces a + // rebuild that re-fires its own exposure. + expect(context.treatment("exp_holdout_added_non_suppressing")).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + const exposures = publisher.publish.mock.calls[0][0].exposures; + const ownExposures = exposures.filter((e) => e.name === "exp_holdout_added_non_suppressing"); + + expect(ownExposures).toHaveLength(2); + + const holdoutExposures = exposures.filter((e) => e.name === "holdout_non_suppressing"); + expect(holdoutExposures).toHaveLength(1); + + done(); + }); + }); + }); + }); + + describe("holdouts: error handling", () => { + // _queueExposure must append to the publish queue and increment pending() BEFORE calling + // the (user-supplied) eventLogger. A throwing eventLogger is isolated by _logEvent itself + // (it never propagates — see _logEvent's doc comment), so this only verifies the exposure + // is queued regardless of the logger throwing. + it("keeps a queued exposure even when the eventLogger throws while reporting it", () => { + const throwingEventLogger = jest.fn((ctx, eventName) => { + if (eventName === "exposure") { + throw new Error("logger boom"); + } + }); + + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_throwing_logger", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 1, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + }, + ], + [] + ); + + const context = new Context( + sdk, + { ...contextOptions, eventLogger: throwingEventLogger }, + contextParams, + response + ); + + expect(() => context.treatment("exp_holdout_throwing_logger")).not.toThrow(); + + // The exposure was appended and counted before the logger ran, and the logger's throw + // was isolated by _logEvent, so it is not lost. + expect(context.pending()).toEqual(1); + }); + + // With N applicable holdouts there are up to N+1 independent exposure-firing attempts per + // call (the covered experiment's own, plus one per holdout). A throwing eventLogger on any + // of them is isolated by _logEvent (it never propagates), so every attempt must still run + // and queue its own exposure regardless of the logger throwing on an earlier one. + it("queues every exposure even when the eventLogger throws while reporting each one", () => { + const eventLogger = jest.fn((ctx, eventName, data) => { + if (eventName === "exposure") { + throw new Error(`boom for ${data.name}`); + } + }); + + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_multi_error", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 1, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11, 12], + }, + ], + [ + { + id: 11, + name: "holdout_multi_error_a", + iteration: 1, + unitType: "session_id", + seedHi: 1, + seedLo: 111, + split: [0.0, 1.0], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + { + id: 12, + name: "holdout_multi_error_b", + iteration: 1, + unitType: "session_id", + seedHi: 2, + seedLo: 222, + split: [0.0, 1.0], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const context = new Context(sdk, { ...contextOptions, eventLogger }, contextParams, response); + + expect(() => context.treatment("exp_holdout_multi_error")).not.toThrow(); + + const exposureCalls = eventLogger.mock.calls.filter((c) => c[1] === "exposure"); + expect(exposureCalls.map((c) => c[2].name).sort()).toEqual( + ["exp_holdout_multi_error", "holdout_multi_error_a", "holdout_multi_error_b"].sort() + ); + + // All three exposures were queued despite all three throwing while reporting. + expect(context.pending()).toEqual(3); + }); + + // A throwing eventLogger (for both "exposure" and "error" events) is isolated by + // _logEvent/_logError themselves (neither ever propagates — see their doc comments), so it + // must not interrupt exposure processing: it must not prevent the holdout loop from running + // after the covered experiment's own exposure is reported, and both exposures must still + // be queued. + it("does not let a throwing error-event logger interrupt exposure processing", () => { + const eventLogger = jest.fn((ctx, eventName, data) => { + if (eventName === "exposure") { + throw new Error(`exposure boom for ${data.name}`); + } + if (eventName === "error") { + throw new Error("error-handler boom"); + } + }); + + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_throwing_error_logger", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 1, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_throwing_error_logger", + iteration: 1, + unitType: "session_id", + seedHi: 1, + seedLo: 111, + split: [0.0, 1.0], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const context = new Context(sdk, { ...contextOptions, eventLogger }, contextParams, response); + + expect(() => context.treatment("exp_holdout_throwing_error_logger")).not.toThrow(); + + const exposureCalls = eventLogger.mock.calls.filter((c) => c[1] === "exposure"); + expect(exposureCalls.map((c) => c[2].name).sort()).toEqual( + ["exp_holdout_throwing_error_logger", "holdout_throwing_error_logger"].sort() + ); + + expect(context.pending()).toEqual(2); + }); + }); + + describe("holdouts: variable resolution", () => { + // Ported from java-sdk's getVariableAssignment (Context.java:1332-1373). A suppressed + // experiment must never win variable-key resolution over one that is genuinely + // assigned/overridden, but it must still be usable as a fallback: a held-out unit reads + // the experiment's own control-variant value, not the caller's default, so it stays + // indistinguishable from a control unit for variable resolution purposes. + const holdoutVarExperiment = (over) => ({ + id: 1, + name: "exp_holdout_var", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: JSON.stringify({ "button.color": "CONTROL_GREY" }) }, + { name: "B", config: JSON.stringify({ "button.color": "TREATMENT_GREEN" }) }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + ...over, + }); + + const alwaysHoldsOut = (over) => ({ + id: 11, + name: "holdout_var", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [1.0, 0.0], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + ...over, + }); + + it("variableValue() returns the control-variant value (not the caller default) for a held-out unit, and still fires the holdout's own exposure", () => { + const response = buildHoldoutResponse([holdoutVarExperiment({})], [alwaysHoldsOut({})]); + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.treatment("exp_holdout_var")).toEqual(0); + expect(context.variableValue("button.color", "APP_FALLBACK")).toEqual("CONTROL_GREY"); + expect(context.pending()).toEqual(1); + }); + + it("peekVariableValue() returns the control-variant value for a held-out unit without triggering any exposure", () => { + const response = buildHoldoutResponse([holdoutVarExperiment({})], [alwaysHoldsOut({})]); + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.peekVariableValue("button.color", "APP_FALLBACK")).toEqual("CONTROL_GREY"); + expect(context.pending()).toEqual(0); + }); + + it("a genuinely assigned experiment still wins variable-key resolution over a suppressed one sharing the same key", () => { + const winningExperiment = { + id: 2, + name: "exp_holdout_var_winner", + iteration: 1, + unitType: "session_id", + seedHi: 300, + seedLo: 400, + split: [0, 1], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 1, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: JSON.stringify({ "button.color": "WINNER_BLUE" }) }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + }; + + const response = buildHoldoutResponse( + [holdoutVarExperiment({}), winningExperiment], + [alwaysHoldsOut({})] + ); + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.variableValue("button.color", "APP_FALLBACK")).toEqual("WINNER_BLUE"); + }); + + it("returns the caller default when every candidate for the key is suppressed but none defines the key on its held-out variant", () => { + const noKeyOnControlExperiment = holdoutVarExperiment({ + variants: [ + { name: "A", config: null }, + { name: "B", config: JSON.stringify({ "button.color": "TREATMENT_GREEN" }) }, + ], + }); + + const response = buildHoldoutResponse([noKeyOnControlExperiment], [alwaysHoldsOut({})]); + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.variableValue("button.color", "APP_FALLBACK")).toEqual("APP_FALLBACK"); + }); + + // Ported from java's getVariableAssignment (Context.java:1362-1364): java pins the whole + // first-encountered suppressed Assignment as the fallback and only checks the key on it + // afterwards — it never skips past a suppressed candidate lacking the key to consider a + // later one that has it. An earlier suppressed candidate without the key must produce the + // caller's default, not a later suppressed candidate's value. + it("an earlier suppressed candidate without the key blocks a later suppressed candidate that has it (java's ordering)", () => { + const firstSuppressed = holdoutVarExperiment({ + id: 1, + name: "exp_holdout_var_first", + // Held out -> forced to variant 0 (control), which does NOT define the key. + variants: [ + { name: "A", config: null }, + { name: "B", config: JSON.stringify({ "button.color": "FIRST_TREATMENT" }) }, + ], + }); + + const secondSuppressed = holdoutVarExperiment({ + id: 2, + name: "exp_holdout_var_second", + // Held out -> forced to variant 0 (control), which DOES define the key. + variants: [ + { name: "A", config: JSON.stringify({ "button.color": "SECOND_CONTROL" }) }, + { name: "B", config: null }, + ], + }); + + const response = buildHoldoutResponse([firstSuppressed, secondSuppressed], [alwaysHoldsOut({})]); + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.variableValue("button.color", "APP_FALLBACK")).toEqual("APP_FALLBACK"); + }); + + // A throwing eventLogger for one candidate's exposure is isolated by _logEvent (it never + // propagates), so it must not stop the loop from visiting (and firing exposures for) the + // remaining candidates sharing the key. + it("a throwing eventLogger for one candidate does not stop later candidates in the loop from being visited and exposed", () => { + const firstExperiment = holdoutVarExperiment({ + id: 1, + name: "exp_holdout_var_throw_first", + holdoutIds: [11], + }); + + const secondExperiment = holdoutVarExperiment({ + id: 2, + name: "exp_holdout_var_throw_second", + holdoutIds: [12], + }); + + const secondHoldout = alwaysHoldsOut({ id: 12, name: "holdout_var_second" }); + + const eventLogger = jest.fn((ctx, eventName, data) => { + if (eventName === "exposure" && data.name === "holdout_var") { + throw new Error("boom"); + } + }); + + const response = buildHoldoutResponse( + [firstExperiment, secondExperiment], + [alwaysHoldsOut({}), secondHoldout] + ); + const context = new Context(sdk, { ...contextOptions, eventLogger }, contextParams, response); + + expect(() => context.variableValue("button.color", "APP_FALLBACK")).not.toThrow(); + + const exposureCalls = eventLogger.mock.calls.filter((c) => c[1] === "exposure"); + expect(exposureCalls.map((c) => c[2].name).sort()).toEqual( + ["holdout_var", "holdout_var_second"].sort() + ); + }); + + // peekVariableValue() shares the exact same suppressedFallback capture as variableValue() + // (both fixed together in the same commit), but never triggers exposures, so it needs its + // own ordering regression test rather than relying on variableValue()'s coverage. + it("peekVariableValue(): an earlier suppressed candidate without the key blocks a later suppressed candidate that has it", () => { + const firstSuppressed = holdoutVarExperiment({ + id: 1, + name: "exp_holdout_var_peek_first", + variants: [ + { name: "A", config: null }, + { name: "B", config: JSON.stringify({ "button.color": "FIRST_TREATMENT" }) }, + ], + }); + + const secondSuppressed = holdoutVarExperiment({ + id: 2, + name: "exp_holdout_var_peek_second", + variants: [ + { name: "A", config: JSON.stringify({ "button.color": "SECOND_CONTROL" }) }, + { name: "B", config: null }, + ], + }); + + const response = buildHoldoutResponse([firstSuppressed, secondSuppressed], [alwaysHoldsOut({})]); + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.peekVariableValue("button.color", "APP_FALLBACK")).toEqual("APP_FALLBACK"); + expect(context.pending()).toEqual(0); + }); + + // A throwing eventLogger for an earlier candidate's exposure is isolated by _logEvent, so + // a later candidate that genuinely wins must still resolve normally and still fire its own + // exposure. + it("resolves to a later winning candidate's value even after an earlier candidate's exposure logger throws", () => { + const suppressedFirst = holdoutVarExperiment({ + id: 1, + name: "exp_holdout_var_winner_after_throw", + holdoutIds: [11], + }); + + const winningSecond = { + id: 2, + name: "exp_holdout_var_winner_after_throw_2", + iteration: 1, + unitType: "session_id", + seedHi: 300, + seedLo: 400, + split: [0, 1], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 1, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: JSON.stringify({ "button.color": "WINNER_AFTER_THROW" }) }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + }; + + const eventLogger = jest.fn((ctx, eventName, data) => { + if (eventName === "exposure" && data.name === "holdout_var") { + throw new Error("boom"); + } + }); + + const response = buildHoldoutResponse([suppressedFirst, winningSecond], [alwaysHoldsOut({})]); + const context = new Context(sdk, { ...contextOptions, eventLogger }, contextParams, response); + + expect(context.variableValue("button.color", "APP_FALLBACK")).toEqual("WINNER_AFTER_THROW"); + + // The winning candidate's own exposure still fired despite the earlier throw. + const exposureCalls = eventLogger.mock.calls.filter((c) => c[1] === "exposure"); + expect(exposureCalls.map((c) => c[2].name)).toContain("exp_holdout_var_winner_after_throw_2"); + }); + + // java's override write path never sets `suppressed` at all (see Assignment.suppressed's + // doc comment), so an override can never become java's suppressedFallback. The JS port + // pins `suppressed` eagerly on the override path too, so the fallback capture must + // explicitly exclude overridden assignments to reproduce the same outcome — otherwise an + // override to a variant that happens to omit the requested key could win the fallback + // slot ahead of a genuinely suppressed sibling that actually defines it. + it("excludes an overridden assignment from the suppressedFallback even when it is also held out", () => { + const overriddenExperiment = { + id: 1, + name: "exp_holdout_var_overridden", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + // Override target (variant 0) omits the key, so this candidate can't win + // outright, and must not be allowed to claim the fallback slot either. + { name: "A", config: null }, + { name: "B", config: JSON.stringify({ "button.color": "OVERRIDE_SIBLING" }) }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }; + + const genuinelySuppressedExperiment = { + id: 2, + name: "exp_holdout_var_genuinely_suppressed", + iteration: 1, + unitType: "session_id", + seedHi: 300, + seedLo: 400, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: JSON.stringify({ "button.color": "GENUINE_SUPPRESSED_CTRL" }) }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [12], + }; + + const secondHoldout = alwaysHoldsOut({ id: 12, name: "holdout_var_second" }); + + const response = buildHoldoutResponse( + [overriddenExperiment, genuinelySuppressedExperiment], + [alwaysHoldsOut({}), secondHoldout] + ); + const context = new Context(sdk, contextOptions, contextParams, response); + context.override("exp_holdout_var_overridden", 0); + + expect(context.variableValue("button.color", "APP_FALLBACK")).toEqual("GENUINE_SUPPRESSED_CTRL"); + }); + + // _peekVariable carries the identical override-exclusion guard as _variableValue (same + // fix, same commit), but never triggers exposures, so it needs its own regression test + // rather than relying on variableValue()'s coverage of the guard. + it("peekVariableValue(): excludes an overridden assignment from the suppressedFallback even when it is also held out", () => { + const overriddenExperiment = { + id: 1, + name: "exp_holdout_var_peek_overridden", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: JSON.stringify({ "button.color": "OVERRIDE_SIBLING" }) }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }; + + const genuinelySuppressedExperiment = { + id: 2, + name: "exp_holdout_var_peek_genuinely_suppressed", + iteration: 1, + unitType: "session_id", + seedHi: 300, + seedLo: 400, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: JSON.stringify({ "button.color": "GENUINE_SUPPRESSED_CTRL" }) }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [12], + }; + + const secondHoldout = alwaysHoldsOut({ id: 12, name: "holdout_var_peek_second" }); + + const response = buildHoldoutResponse( + [overriddenExperiment, genuinelySuppressedExperiment], + [alwaysHoldsOut({}), secondHoldout] + ); + const context = new Context(sdk, contextOptions, contextParams, response); + context.override("exp_holdout_var_peek_overridden", 0); + + expect(context.peekVariableValue("button.color", "APP_FALLBACK")).toEqual("GENUINE_SUPPRESSED_CTRL"); + expect(context.pending()).toEqual(0); + }); + }); + }); + describe("variableValue()", () => { it("should not return variable values when unassigned", (done) => { const context = new Context(sdk, contextOptions, contextParams, audienceStrictContextResponse); diff --git a/src/context.ts b/src/context.ts index 8267382..62c9348 100644 --- a/src/context.ts +++ b/src/context.ts @@ -47,6 +47,7 @@ export type ExperimentData = { custom: boolean; audienceMismatch: boolean; customFieldValues: CustomFieldValue[] | null; + holdoutIds?: number[]; }; type Assignment = { @@ -68,11 +69,33 @@ type Assignment = { trafficSplit?: number[]; variables?: Record; attrsSeq?: number; + suppressed?: boolean; + holdouts?: HoldoutExperiment[]; + holdoutAssignments?: (Assignment | null)[]; + // `split.length` at the time `variant` was resolved; a same-iteration refresh can change + // the live value without invalidating the cache. + holdoutArmCount?: number; +}; + +export type HoldoutData = { + id: number; + name: string; + unitType?: string | null; + iteration: number; + seedHi: number; + seedLo: number; + split: number[]; + holdoutType?: string; +}; + +export type HoldoutExperiment = { + data: HoldoutData; }; export type Experiment = { data: ExperimentData; variables: Record[]; + holdouts?: HoldoutExperiment[] | null; }; export type Unit = { @@ -126,8 +149,15 @@ export type ContextOptions = { export type ContextData = { experiments?: ExperimentData[]; + holdouts?: HoldoutData[]; }; +function isHeldOutBy(holdoutVariant: number, holdoutArmCount: number, fullOnVariant: number): boolean { + if (holdoutVariant === 0) return true; + if (holdoutArmCount === 3 && holdoutVariant === 1) return fullOnVariant === 0; + return false; +} + export default class Context { private readonly _assigners: Record; private readonly _attrs: Attribute[]; @@ -150,6 +180,8 @@ export default class Context { private _goals: Goal[]; private _index: Record; private _indexVariables: Record; + private _holdoutsById: Record; + private _holdoutAssignments: Record; private _overrides: Record; private _pending: number; private _attrsSeq: number; @@ -178,6 +210,7 @@ export default class Context { this._cassignments = {}; this._units = {}; this._assigners = {}; + this._holdoutAssignments = {}; this._audienceMatcher = new AudienceMatcher(); this._environmentName = null; this._attrsSeq = 0; @@ -319,6 +352,26 @@ export default class Context { } this._units[unitType] = uid; + + this._invalidateAssignmentsPinnedWithMissingUnit(unitType); + } + + // A null holdout entry means the unit was missing when the assignment was resolved. Evict so + // `_assign()` rebuilds it; already-exposed ones are kept since re-resolving would queue a + // second, conflicting exposure. + private _invalidateAssignmentsPinnedWithMissingUnit(unitType: string): void { + for (const experimentName in this._assignments) { + const assignment = this._assignments[experimentName]; + const holdoutAssignments = assignment.holdoutAssignments; + + if (holdoutAssignments && assignment.unitType === unitType && !assignment.exposed) { + const hasMissingEntry = holdoutAssignments.some((holdoutAssignment) => holdoutAssignment === null); + + if (hasMissingEntry) { + delete this._assignments[experimentName]; + } + } + } } getUnits() { @@ -565,6 +618,27 @@ export default class Context { return true; }; + // (id, iteration) only: seed/split edits must not force a duplicate exposure. + const holdoutSetMatches = (experiment: Experiment, assignment: Assignment) => { + const freshHoldouts = experiment.holdouts ?? []; + const pinnedHoldouts = assignment.holdouts ?? []; + + if (freshHoldouts.length !== pinnedHoldouts.length) { + return false; + } + + for (let i = 0; i < freshHoldouts.length; i++) { + if (freshHoldouts[i].data.id !== pinnedHoldouts[i].data.id) { + return false; + } + if (freshHoldouts[i].data.iteration !== pinnedHoldouts[i].data.iteration) { + return false; + } + } + + return true; + }; + const hasCustom = experimentName in this._cassignments; const hasOverride = experimentName in this._overrides; const experiment = experimentName in this._index ? this._index[experimentName] : null; @@ -572,7 +646,11 @@ export default class Context { if (experimentName in this._assignments) { const assignment = this._assignments[experimentName]; if (hasOverride) { - if (assignment.overridden && assignment.variant === this._overrides[experimentName]) { + if ( + assignment.overridden && + assignment.variant === this._overrides[experimentName] && + (experiment == null || holdoutSetMatches(experiment, assignment)) + ) { // override up-to-date return assignment; } @@ -581,8 +659,13 @@ export default class Context { // previously not-running experiment return assignment; } - } else if (!hasCustom || this._cassignments[experimentName] === assignment.variant) { - if (experimentMatches(experiment.data, assignment) && audienceMatches(experiment.data, assignment)) { + } else if (assignment.suppressed || !hasCustom || this._cassignments[experimentName] === assignment.variant) { + // Suppressed assignments are forced to variant 0, so a custom mismatch is not staleness. + if ( + experimentMatches(experiment.data, assignment) && + audienceMatches(experiment.data, assignment) && + holdoutSetMatches(experiment, assignment) + ) { // assignment up-to-date return assignment; } @@ -607,6 +690,25 @@ export default class Context { this._assignments[experimentName] = assignment; + // Resolved for overrides too: holdout exposures fire from `holdoutAssignments` regardless. + if (experiment != null && experiment.holdouts != null && experiment.holdouts.length > 0) { + const holdouts = experiment.holdouts; + const holdoutUnitType = experiment.data.unitType; + + const holdoutAssignments: (Assignment | null)[] = holdouts.map((holdout) => + holdoutUnitType !== null ? this._getHoldoutAssignment(holdout, holdoutUnitType) : null + ); + + assignment.holdouts = holdouts; + assignment.holdoutAssignments = holdoutAssignments; + + assignment.suppressed = holdoutAssignments.some( + (holdoutAssignment) => + holdoutAssignment != null && + isHeldOutBy(holdoutAssignment.variant, holdoutAssignment.holdoutArmCount ?? 0, experiment.data.fullOnVariant) + ); + } + if (hasOverride) { if (experiment != null) { assignment.id = experiment.data.id; @@ -618,9 +720,14 @@ export default class Context { } else { if (experiment != null) { const unitType = experiment.data.unitType; + const attrs = this._getAttributesMap(); + + // Set even when suppressed, or `audienceMatches` rebuilds (losing `exposed`) on every call. + assignment.ruleKey = experiment.data.assignmentRules + ? `${experiment.data.assignmentRules}:${this._environmentName}` + : ""; let ruleVariant: number | null = null; - const attrs = this._getAttributesMap(); if (experiment.data.assignmentRules && experiment.data.assignmentRules.length > 0) { ruleVariant = this._computeRuleVariant( @@ -631,22 +738,22 @@ export default class Context { } assignment.ruleVariant = ruleVariant; - assignment.ruleKey = experiment.data.assignmentRules - ? `${experiment.data.assignmentRules}:${this._environmentName}` - : ""; + // A matching assignment rule wins over holdout suppression, the same way override() + // does: it's an explicit, author-specified assignment (flagged `ruleOverride`, + // excluded from stats) rather than the experiment's own randomized/custom path, so + // suppression only applies when no rule matched. if (ruleVariant !== null) { assignment.variant = ruleVariant; assignment.ruleOverride = true; + } else if (assignment.suppressed) { + assignment.assigned = false; + assignment.variant = 0; } else { if (experiment.data.audience && experiment.data.audience.length > 0) { - const result = this._evaluateAudience(experiment.data.audience); + const result = this._audienceMatcher.evaluate(experiment.data.audience, attrs); - // Only flag a mismatch when the audience actually evaluated - // to a boolean. A null result (e.g. an audience with no - // usable filter like `{}`) leaves audienceMismatch false, - // matching the collector (ContextAPI: `if (result != null)`). - if (result !== null) { + if (typeof result === "boolean") { assignment.audienceMismatch = !result; } } @@ -724,12 +831,38 @@ export default class Context { if (!assignment.exposed) { assignment.exposed = true; - this._queueExposure(experimentName, assignment); + this._triggerExposures(experimentName, assignment); } return assignment; } + private _triggerExposures(experimentName: string, assignment: Assignment): void { + if (!assignment.suppressed || assignment.overridden || assignment.ruleOverride) { + this._queueExposure(experimentName, assignment); + } + + this._triggerApplicableHoldoutExposures(assignment); + } + + // Uses the pinned snapshot, not a live re-resolution, so a refresh between the suppression + // decision and here can't publish a holdout exposure from a different epoch. + private _triggerApplicableHoldoutExposures(assignment: Assignment): void { + const holdouts = assignment.holdouts; + const holdoutAssignments = assignment.holdoutAssignments; + if (!holdouts || !holdoutAssignments) return; + + holdoutAssignments.forEach((holdoutAssignment, i) => { + if (holdoutAssignment == null) return; + + if (!holdoutAssignment.exposed) { + holdoutAssignment.exposed = true; + + this._queueExposure(holdouts[i].data.name, holdoutAssignment); + } + }); + } + private _queueExposure(experimentName: string, assignment: Assignment) { const exposureEvent: Exposure = { id: assignment.id, @@ -745,12 +878,17 @@ export default class Context { audienceMismatch: assignment.audienceMismatch, ruleOverride: assignment.ruleOverride, }; - this._logEvent("exposure", exposureEvent); - + // The exposure is appended and counted BEFORE the (user-supplied) eventLogger runs: a + // throwing logger must not discard the exposure itself, only fail to report it. _setTimeout + // is scheduled in a finally so a throwing logger still flushes what's already queued. this._exposures.push(exposureEvent); this._pending++; - this._setTimeout(); + try { + this._logEvent("exposure", exposureEvent); + } finally { + this._setTimeout(); + } } private _customFieldKeys() { @@ -843,22 +981,40 @@ export default class Context { return this._customFieldValueType(experimentName, key); } + // A held-out unit reads the control value, not the caller's default, when nothing else wins. + // The first suppressed candidate is captured even if it lacks the key, so a later one can't win. private _resolveVariableValue(key: string, defaultValue: string, shouldQueueExposure: boolean): string { + let suppressedFallback: Record | undefined; + for (const experiment of this._indexVariables[key] ?? []) { const experimentName = experiment.data.name; const assignment = this._assign(experimentName); if (assignment.variables !== undefined) { if (shouldQueueExposure && !assignment.exposed) { assignment.exposed = true; - this._queueExposure(experimentName, assignment); + + this._triggerExposures(experimentName, assignment); } if (key in assignment.variables && (assignment.assigned || assignment.overridden || assignment.ruleOverride)) { return assignment.variables[key] as string; } + + if ( + assignment.suppressed && + !assignment.overridden && + !assignment.ruleOverride && + suppressedFallback === undefined + ) { + suppressedFallback = assignment.variables; + } } } + if (suppressedFallback !== undefined && key in suppressedFallback) { + return suppressedFallback[key] as string; + } + return defaultValue; } @@ -1204,7 +1360,12 @@ export default class Context { } if (!(unitType in this._hashes)) { - const hash = unitType in this._units ? hashUnit(this._units[unitType]) : null; + // Never cache `null`: the unit can still be set later (see `_getHoldoutAssignment`). + if (!(unitType in this._units)) { + return null; + } + + const hash = hashUnit(this._units[unitType]); this._hashes[unitType] = hash; return hash; } @@ -1212,6 +1373,46 @@ export default class Context { return this._hashes[unitType]; } + private _getHoldoutAssignment(holdout: HoldoutExperiment, unitType: string): Assignment | null { + // `holdout` may predate a refresh; prefer the live definition. + const liveHoldoutData = this._holdoutsById[holdout.data.id] ?? holdout.data; + + const cacheKey = `${liveHoldoutData.id}:${unitType}`; + const cached = this._holdoutAssignments[cacheKey]; + if (cached && cached.id === liveHoldoutData.id && cached.iteration === liveHoldoutData.iteration) { + return cached; + } + + const unit = this._unitHash(unitType); + if (unit === null) { + return null; + } + + const assigner = + unitType in this._assigners ? this._assigners[unitType] : (this._assigners[unitType] = new VariantAssigner(unit)); + + const assignment: Assignment = { + id: liveHoldoutData.id, + iteration: liveHoldoutData.iteration, + fullOnVariant: 0, + unitType, + variant: assigner.assign(liveHoldoutData.split, liveHoldoutData.seedHi, liveHoldoutData.seedLo), + overridden: false, + assigned: true, + exposed: false, + eligible: true, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + holdoutArmCount: liveHoldoutData.split.length, + }; + + this._holdoutAssignments[cacheKey] = assignment; + + return assignment; + } + private _init(data: ContextData, assignments: Record = {}) { this._data = data; this._environmentName = this._sdk.getClient().getEnvironment() ?? null; @@ -1219,11 +1420,57 @@ export default class Context { const index: Record = {}; const indexVariables: Record = {}; - for (const experiment of data.experiments || []) { + const holdoutsById: Record = {}; + + (data.holdouts || []).forEach((holdout) => { + if (holdout.split && holdout.split.length > 0) { + holdoutsById[holdout.id] = holdout; + } + }); + + this._holdoutsById = holdoutsById; + + const holdoutExperiments: Record = {}; + + const resolveHoldoutExperiment = (holdoutId: number): HoldoutExperiment | undefined => { + if (holdoutExperiments[holdoutId]) { + return holdoutExperiments[holdoutId]; + } + + const holdoutData = this._holdoutsById[holdoutId]; + if (!holdoutData) { + return undefined; + } + + const holdoutEntry: HoldoutExperiment = { + data: holdoutData, + }; + + holdoutExperiments[holdoutId] = holdoutEntry; + return holdoutEntry; + }; + + (data.experiments || []).forEach((experiment) => { const variables: Record[] = []; - const entry = { + + let holdouts: HoldoutExperiment[] | null = null; + if (experiment.holdoutIds && experiment.holdoutIds.length > 0) { + const resolved: HoldoutExperiment[] = []; + + experiment.holdoutIds.forEach((holdoutId) => { + const holdoutExperiment = resolveHoldoutExperiment(holdoutId); + if (holdoutExperiment) { + insertUniqueSorted(resolved, holdoutExperiment, (a, b) => a.data.id < b.data.id); + } + }); + + holdouts = resolved.length > 0 ? resolved : null; + } + + const entry: Experiment = { data: experiment, variables, + holdouts, }; index[experiment.name] = entry; @@ -1261,7 +1508,7 @@ export default class Context { variables[i] = parsed; } - } + }); this._index = index; this._indexVariables = indexVariables;